1
2
3
4 """
5 This file is part of the web2py Web Framework
6 Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu>
7 License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)
8
9 Thanks to
10 * Niall Sweeny <niall.sweeny@fonjax.com> for MS SQL support
11 * Marcel Leuthi <mluethi@mlsystems.ch> for Oracle support
12 * Denes
13 * Chris Clark
14 * clach05
15 * Denes Lengyel
16 * and many others who have contributed to current and previous versions
17
18 This file contains the DAL support for many relational databases,
19 including:
20 - SQLite & SpatiaLite
21 - MySQL
22 - Postgres
23 - Firebird
24 - Oracle
25 - MS SQL
26 - DB2
27 - Interbase
28 - Ingres
29 - Informix (9+ and SE)
30 - SapDB (experimental)
31 - Cubrid (experimental)
32 - CouchDB (experimental)
33 - MongoDB (in progress)
34 - Google:nosql
35 - Google:sql
36 - Teradata
37 - IMAP (experimental)
38
39 Example of usage:
40
41 >>> # from dal import DAL, Field
42
43 ### create DAL connection (and create DB if it doesn't exist)
44 >>> db = DAL(('sqlite://storage.sqlite','mysql://a:b@localhost/x'),
45 ... folder=None)
46
47 ### define a table 'person' (create/alter as necessary)
48 >>> person = db.define_table('person',Field('name','string'))
49
50 ### insert a record
51 >>> id = person.insert(name='James')
52
53 ### retrieve it by id
54 >>> james = person(id)
55
56 ### retrieve it by name
57 >>> james = person(name='James')
58
59 ### retrieve it by arbitrary query
60 >>> query = (person.name=='James') & (person.name.startswith('J'))
61 >>> james = db(query).select(person.ALL)[0]
62
63 ### update one record
64 >>> james.update_record(name='Jim')
65 <Row {'id': 1, 'name': 'Jim'}>
66
67 ### update multiple records by query
68 >>> db(person.name.like('J%')).update(name='James')
69 1
70
71 ### delete records by query
72 >>> db(person.name.lower() == 'jim').delete()
73 0
74
75 ### retrieve multiple records (rows)
76 >>> people = db(person).select(orderby=person.name,
77 ... groupby=person.name, limitby=(0,100))
78
79 ### further filter them
80 >>> james = people.find(lambda row: row.name == 'James').first()
81 >>> print james.id, james.name
82 1 James
83
84 ### check aggregates
85 >>> counter = person.id.count()
86 >>> print db(person).select(counter).first()(counter)
87 1
88
89 ### delete one record
90 >>> james.delete_record()
91 1
92
93 ### delete (drop) entire database table
94 >>> person.drop()
95
96 Supported field types:
97 id string text boolean integer double decimal password upload
98 blob time date datetime
99
100 Supported DAL URI strings:
101 'sqlite://test.db'
102 'spatialite://test.db'
103 'sqlite:memory'
104 'spatialite:memory'
105 'jdbc:sqlite://test.db'
106 'mysql://root:none@localhost/test'
107 'postgres://mdipierro:password@localhost/test'
108 'postgres:psycopg2://mdipierro:password@localhost/test'
109 'postgres:pg8000://mdipierro:password@localhost/test'
110 'jdbc:postgres://mdipierro:none@localhost/test'
111 'mssql://web2py:none@A64X2/web2py_test'
112 'mssql2://web2py:none@A64X2/web2py_test' # alternate mappings
113 'oracle://username:password@database'
114 'firebird://user:password@server:3050/database'
115 'db2://DSN=dsn;UID=user;PWD=pass'
116 'firebird://username:password@hostname/database'
117 'firebird_embedded://username:password@c://path'
118 'informix://user:password@server:3050/database'
119 'informixu://user:password@server:3050/database' # unicode informix
120 'ingres://database' # or use an ODBC connection string, e.g. 'ingres://dsn=dsn_name'
121 'google:datastore' # for google app engine datastore
122 'google:sql' # for google app engine with sql (mysql compatible)
123 'teradata://DSN=dsn;UID=user;PWD=pass; DATABASE=database' # experimental
124 'imap://user:password@server:port' # experimental
125 'mongodb://user:password@server:port/database' # experimental
126
127 For more info:
128 help(DAL)
129 help(Field)
130 """
131
132
133
134
135
136 __all__ = ['DAL', 'Field']
137
138 DEFAULTLENGTH = {'string':512,
139 'password':512,
140 'upload':512,
141 'text':2**15,
142 'blob':2**31}
143 TIMINGSSIZE = 100
144 SPATIALLIBS = {
145 'Windows':'libspatialite',
146 'Linux':'libspatialite.so',
147 'Darwin':'libspatialite.dylib'
148 }
149 DEFAULT_URI = 'sqlite://dummy.db'
150
151 import re
152 import sys
153 import locale
154 import os
155 import types
156 import datetime
157 import threading
158 import time
159 import csv
160 import cgi
161 import copy
162 import socket
163 import logging
164 import base64
165 import shutil
166 import marshal
167 import decimal
168 import struct
169 import urllib
170 import hashlib
171 import uuid
172 import glob
173 import traceback
174 import platform
175
176 PYTHON_VERSION = sys.version_info[0]
177 if PYTHON_VERSION == 2:
178 import cPickle as pickle
179 import cStringIO as StringIO
180 import copy_reg as copyreg
181 hashlib_md5 = hashlib.md5
182 bytes, unicode = str, unicode
183 else:
184 import pickle
185 from io import StringIO as StringIO
186 import copyreg
187 long = int
188 hashlib_md5 = lambda s: hashlib.md5(bytes(s,'utf8'))
189 bytes, unicode = bytes, str
190
191 CALLABLETYPES = (types.LambdaType, types.FunctionType,
192 types.BuiltinFunctionType,
193 types.MethodType, types.BuiltinMethodType)
194
195 TABLE_ARGS = set(
196 ('migrate','primarykey','fake_migrate','format','redefine',
197 'singular','plural','trigger_name','sequence_name','fields',
198 'common_filter','polymodel','table_class','on_define','actual_name'))
199
200 SELECT_ARGS = set(
201 ('orderby', 'groupby', 'limitby','required', 'cache', 'left',
202 'distinct', 'having', 'join','for_update', 'processor','cacheable', 'orderby_on_limitby'))
203
204 ogetattr = object.__getattribute__
205 osetattr = object.__setattr__
206 exists = os.path.exists
207 pjoin = os.path.join
208
209
210
211
212 try:
213 from gluon.utils import web2py_uuid
214 except (ImportError, SystemError):
215 import uuid
217
218 try:
219 import portalocker
220 have_portalocker = True
221 except ImportError:
222 have_portalocker = False
223
224 try:
225 from gluon import serializers
226 have_serializers = True
227 except ImportError:
228 have_serializers = False
229 try:
230 import json as simplejson
231 except ImportError:
232 try:
233 import gluon.contrib.simplejson as simplejson
234 except ImportError:
235 simplejson = None
236
237 LOGGER = logging.getLogger("web2py.dal")
238 DEFAULT = lambda:0
239
240 GLOBAL_LOCKER = threading.RLock()
241 THREAD_LOCAL = threading.local()
242
243
244
245
246 REGEX_TYPE = re.compile('^([\w\_\:]+)')
247 REGEX_DBNAME = re.compile('^(\w+)(\:\w+)*')
248 REGEX_W = re.compile('^\w+$')
249 REGEX_TABLE_DOT_FIELD = re.compile('^(\w+)\.(\w+)$')
250 REGEX_UPLOAD_PATTERN = re.compile('(?P<table>[\w\-]+)\.(?P<field>[\w\-]+)\.(?P<uuidkey>[\w\-]+)(\.(?P<name>\w+))?\.\w+$')
251 REGEX_CLEANUP_FN = re.compile('[\'"\s;]+')
252 REGEX_UNPACK = re.compile('(?<!\|)\|(?!\|)')
253 REGEX_PYTHON_KEYWORDS = re.compile('^(and|del|from|not|while|as|elif|global|or|with|assert|else|if|pass|yield|break|except|import|print|class|exec|in|raise|continue|finally|is|return|def|for|lambda|try)$')
254 REGEX_SELECT_AS_PARSER = re.compile("\s+AS\s+(\S+)")
255 REGEX_CONST_STRING = re.compile('(\"[^\"]*?\")|(\'[^\']*?\')')
256 REGEX_SEARCH_PATTERN = re.compile('^{[^\.]+\.[^\.]+(\.(lt|gt|le|ge|eq|ne|contains|startswith|year|month|day|hour|minute|second))?(\.not)?}$')
257 REGEX_SQUARE_BRACKETS = re.compile('^.+\[.+\]$')
258 REGEX_STORE_PATTERN = re.compile('\.(?P<e>\w{1,5})$')
259 REGEX_QUOTES = re.compile("'[^']*'")
260 REGEX_ALPHANUMERIC = re.compile('^[0-9a-zA-Z]\w*$')
261 REGEX_PASSWORD = re.compile('\://([^:@]*)\:')
262 REGEX_NOPASSWD = re.compile('\/\/[\w\.\-]+[\:\/](.+)(?=@)')
263
264
265
266 DRIVERS = []
267
268 try:
269 from new import classobj
270 from google.appengine.ext import db as gae
271 from google.appengine.api import namespace_manager, rdbms
272 from google.appengine.api.datastore_types import Key
273 from google.appengine.ext.db.polymodel import PolyModel
274 DRIVERS.append('google')
275 except ImportError:
276 pass
277
278 if not 'google' in DRIVERS:
279
280 try:
281 from pysqlite2 import dbapi2 as sqlite2
282 DRIVERS.append('SQLite(sqlite2)')
283 except ImportError:
284 LOGGER.debug('no SQLite drivers pysqlite2.dbapi2')
285
286 try:
287 from sqlite3 import dbapi2 as sqlite3
288 DRIVERS.append('SQLite(sqlite3)')
289 except ImportError:
290 LOGGER.debug('no SQLite drivers sqlite3')
291
292 try:
293
294 try:
295 import gluon.contrib.pymysql as pymysql
296
297
298 pymysql.ESCAPE_REGEX = re.compile("'")
299 pymysql.ESCAPE_MAP = {"'": "''"}
300
301 except ImportError:
302 import pymysql
303 DRIVERS.append('MySQL(pymysql)')
304 except ImportError:
305 LOGGER.debug('no MySQL driver pymysql')
306
307 try:
308 import MySQLdb
309 DRIVERS.append('MySQL(MySQLdb)')
310 except ImportError:
311 LOGGER.debug('no MySQL driver MySQLDB')
312
313 try:
314 import mysql.connector as mysqlconnector
315 DRIVERS.append("MySQL(mysqlconnector)")
316 except ImportError:
317 LOGGER.debug("no driver mysql.connector")
318
319 try:
320 import psycopg2
321 from psycopg2.extensions import adapt as psycopg2_adapt
322 DRIVERS.append('PostgreSQL(psycopg2)')
323 except ImportError:
324 LOGGER.debug('no PostgreSQL driver psycopg2')
325
326 try:
327
328 try:
329 import gluon.contrib.pg8000.dbapi as pg8000
330 except ImportError:
331 import pg8000.dbapi as pg8000
332 DRIVERS.append('PostgreSQL(pg8000)')
333 except ImportError:
334 LOGGER.debug('no PostgreSQL driver pg8000')
335
336 try:
337 import cx_Oracle
338 DRIVERS.append('Oracle(cx_Oracle)')
339 except ImportError:
340 LOGGER.debug('no Oracle driver cx_Oracle')
341
342 try:
343 try:
344 import pyodbc
345 except ImportError:
346 try:
347 import gluon.contrib.pypyodbc as pyodbc
348 except Exception, e:
349 raise ImportError(str(e))
350 DRIVERS.append('MSSQL(pyodbc)')
351 DRIVERS.append('DB2(pyodbc)')
352 DRIVERS.append('Teradata(pyodbc)')
353 DRIVERS.append('Ingres(pyodbc)')
354 except ImportError:
355 LOGGER.debug('no MSSQL/DB2/Teradata/Ingres driver pyodbc')
356
357 try:
358 import Sybase
359 DRIVERS.append('Sybase(Sybase)')
360 except ImportError:
361 LOGGER.debug('no Sybase driver')
362
363 try:
364 import kinterbasdb
365 DRIVERS.append('Interbase(kinterbasdb)')
366 DRIVERS.append('Firebird(kinterbasdb)')
367 except ImportError:
368 LOGGER.debug('no Firebird/Interbase driver kinterbasdb')
369
370 try:
371 import fdb
372 DRIVERS.append('Firebird(fdb)')
373 except ImportError:
374 LOGGER.debug('no Firebird driver fdb')
375
376 try:
377 import firebirdsql
378 DRIVERS.append('Firebird(firebirdsql)')
379 except ImportError:
380 LOGGER.debug('no Firebird driver firebirdsql')
381
382 try:
383 import informixdb
384 DRIVERS.append('Informix(informixdb)')
385 LOGGER.warning('Informix support is experimental')
386 except ImportError:
387 LOGGER.debug('no Informix driver informixdb')
388
389 try:
390 import sapdb
391 DRIVERS.append('SQL(sapdb)')
392 LOGGER.warning('SAPDB support is experimental')
393 except ImportError:
394 LOGGER.debug('no SAP driver sapdb')
395
396 try:
397 import cubriddb
398 DRIVERS.append('Cubrid(cubriddb)')
399 LOGGER.warning('Cubrid support is experimental')
400 except ImportError:
401 LOGGER.debug('no Cubrid driver cubriddb')
402
403 try:
404 from com.ziclix.python.sql import zxJDBC
405 import java.sql
406
407 from org.sqlite import JDBC
408 zxJDBC_sqlite = java.sql.DriverManager
409 DRIVERS.append('PostgreSQL(zxJDBC)')
410 DRIVERS.append('SQLite(zxJDBC)')
411 LOGGER.warning('zxJDBC support is experimental')
412 is_jdbc = True
413 except ImportError:
414 LOGGER.debug('no SQLite/PostgreSQL driver zxJDBC')
415 is_jdbc = False
416
417 try:
418 import couchdb
419 DRIVERS.append('CouchDB(couchdb)')
420 except ImportError:
421 LOGGER.debug('no Couchdb driver couchdb')
422
423 try:
424 import pymongo
425 DRIVERS.append('MongoDB(pymongo)')
426 except:
427 LOGGER.debug('no MongoDB driver pymongo')
428
429 try:
430 import imaplib
431 DRIVERS.append('IMAP(imaplib)')
432 except:
433 LOGGER.debug('no IMAP driver imaplib')
434
435 PLURALIZE_RULES = [
436 (re.compile('child$'), re.compile('child$'), 'children'),
437 (re.compile('oot$'), re.compile('oot$'), 'eet'),
438 (re.compile('ooth$'), re.compile('ooth$'), 'eeth'),
439 (re.compile('l[eo]af$'), re.compile('l([eo])af$'), 'l\\1aves'),
440 (re.compile('sis$'), re.compile('sis$'), 'ses'),
441 (re.compile('man$'), re.compile('man$'), 'men'),
442 (re.compile('ife$'), re.compile('ife$'), 'ives'),
443 (re.compile('eau$'), re.compile('eau$'), 'eaux'),
444 (re.compile('lf$'), re.compile('lf$'), 'lves'),
445 (re.compile('[sxz]$'), re.compile('$'), 'es'),
446 (re.compile('[^aeioudgkprt]h$'), re.compile('$'), 'es'),
447 (re.compile('(qu|[^aeiou])y$'), re.compile('y$'), 'ies'),
448 (re.compile('$'), re.compile('$'), 's'),
449 ]
456
461
464
467
469
472
474 regex = re.compile('\.keyword(?=\w)')
475 a = regex.sub('."%s"' % keyword,a)
476 return a
477
478 if 'google' in DRIVERS:
479
480 is_jdbc = False
483 """
484 GAE decimal implementation
485 """
486 data_type = decimal.Decimal
487
488 - def __init__(self, precision, scale, **kwargs):
489 super(GAEDecimalProperty, self).__init__(self, **kwargs)
490 d = '1.'
491 for x in range(scale):
492 d += '0'
493 self.round = decimal.Decimal(d)
494
502
504 if value is None or value == '':
505 return None
506 else:
507 return decimal.Decimal(value).quantize(self.round)
508
510 value = super(GAEDecimalProperty, self).validate(value)
511 if value is None or isinstance(value, decimal.Decimal):
512 return value
513 elif isinstance(value, basestring):
514 return decimal.Decimal(value)
515 raise gae.BadValueError("Property %s must be a Decimal or string."\
516 % self.name)
517
523
524 POOLS = {}
525 check_active_connection = True
526
527 @staticmethod
530
531
532
533 - def close(self,action='commit',really=True):
550
551 @staticmethod
553 """ to close cleanly databases in a multithreaded environment """
554 dbs = getattr(THREAD_LOCAL,'db_instances',{}).items()
555 for db_uid, db_group in dbs:
556 for db in db_group:
557 if hasattr(db,'_adapter'):
558 db._adapter.close(action)
559 getattr(THREAD_LOCAL,'db_instances',{}).clear()
560 getattr(THREAD_LOCAL,'db_instances_zombie',{}).clear()
561 if callable(action):
562 action(None)
563 return
564
577
579 """hook for the after_connection parameter"""
580 if callable(self._after_connection):
581 self._after_connection(self)
582 self.after_connection()
583
585 """ this it is supposed to be overloaded by adapters"""
586 pass
587
589 """
590 this function defines: self.connection and self.cursor
591 (iff cursor is True)
592 if self.pool_size>0 it will try pull the connection from the pool
593 if the connection is not active (closed by db server) it will loop
594 if not self.pool_size or no active connections in pool makes a new one
595 """
596 if getattr(self,'connection', None) != None:
597 return
598 if f is None:
599 f = self.connector
600
601
602
603
604
605 if not self.pool_size:
606 self.connection = f()
607 self.cursor = cursor and self.connection.cursor()
608 else:
609 uri = self.uri
610 POOLS = ConnectionPool.POOLS
611 while True:
612 GLOBAL_LOCKER.acquire()
613 if not uri in POOLS:
614 POOLS[uri] = []
615 if POOLS[uri]:
616 self.connection = POOLS[uri].pop()
617 GLOBAL_LOCKER.release()
618 self.cursor = cursor and self.connection.cursor()
619 try:
620 if self.cursor and self.check_active_connection:
621 self.execute('SELECT 1;')
622 break
623 except:
624 pass
625 else:
626 GLOBAL_LOCKER.release()
627 self.connection = f()
628 self.cursor = cursor and self.connection.cursor()
629 break
630 self.after_connection_hook()
631
638 native_json = False
639 driver = None
640 driver_name = None
641 drivers = ()
642 connection = None
643 commit_on_alter_table = False
644 support_distributed_transaction = False
645 uploads_in_blob = False
646 can_select_for_update = True
647 dbpath = None
648 folder = None
649
650 TRUE = 'T'
651 FALSE = 'F'
652 T_SEP = ' '
653 QUOTE_TEMPLATE = '"%s"'
654
655 types = {
656 'boolean': 'CHAR(1)',
657 'string': 'CHAR(%(length)s)',
658 'text': 'TEXT',
659 'json': 'TEXT',
660 'password': 'CHAR(%(length)s)',
661 'blob': 'BLOB',
662 'upload': 'CHAR(%(length)s)',
663 'integer': 'INTEGER',
664 'bigint': 'INTEGER',
665 'float':'DOUBLE',
666 'double': 'DOUBLE',
667 'decimal': 'DOUBLE',
668 'date': 'DATE',
669 'time': 'TIME',
670 'datetime': 'TIMESTAMP',
671 'id': 'INTEGER PRIMARY KEY AUTOINCREMENT',
672 'reference': 'INTEGER REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
673 'list:integer': 'TEXT',
674 'list:string': 'TEXT',
675 'list:reference': 'TEXT',
676
677 'big-id': 'BIGINT PRIMARY KEY AUTOINCREMENT',
678 'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
679 }
680
682 if not hasattr(self.driver, "OperationalError"):
683 return None
684 return isinstance(exception, self.driver.OperationalError)
685
687 if not hasattr(self.driver, "ProgrammingError"):
688 return None
689 return isinstance(exception, self.driver.ProgrammingError)
690
692 pkeys = getattr(table,'_primarykey',None)
693 if pkeys:
694 return table[pkeys[0]] != None
695 else:
696 return table._id != None
697
699 return "'%s'" % obj.replace("'", "''")
700
702 if isinstance(obj,(int,float)):
703 return str(obj)
704 return self.adapt(str(obj))
705
707 """
708 to be used ONLY for files that on GAE may not be on filesystem
709 """
710 return exists(filename)
711
712 - def file_open(self, filename, mode='rb', lock=True):
713 """
714 to be used ONLY for files that on GAE may not be on filesystem
715 """
716 if have_portalocker and lock:
717 fileobj = portalocker.LockedFile(filename,mode)
718 else:
719 fileobj = open(filename,mode)
720 return fileobj
721
723 """
724 to be used ONLY for files that on GAE may not be on filesystem
725 """
726 if fileobj:
727 fileobj.close()
728
731
733 self.adapter_args = adapter_args
734 if getattr(self,'driver',None) != None:
735 return
736 drivers_available = [driver for driver in self.drivers
737 if driver in globals()]
738 if uri:
739 items = uri.split('://',1)[0].split(':')
740 request_driver = items[1] if len(items)>1 else None
741 else:
742 request_driver = None
743 request_driver = request_driver or adapter_args.get('driver')
744 if request_driver:
745 if request_driver in drivers_available:
746 self.driver_name = request_driver
747 self.driver = globals().get(request_driver)
748 else:
749 raise RuntimeError("driver %s not available" % request_driver)
750 elif drivers_available:
751 self.driver_name = drivers_available[0]
752 self.driver = globals().get(self.driver_name)
753 else:
754 raise RuntimeError("no driver available %s" % str(self.drivers))
755
756 - def log(self, message, table=None):
757 """ Logs migrations
758
759 It will not log changes if logfile is not specified. Defaults
760 to sql.log
761 """
762
763 isabs = None
764 logfilename = self.adapter_args.get('logfile','sql.log')
765 writelog = bool(logfilename)
766 if writelog:
767 isabs = os.path.isabs(logfilename)
768
769 if table and table._dbt and writelog and self.folder:
770 if isabs:
771 table._loggername = logfilename
772 else:
773 table._loggername = pjoin(self.folder, logfilename)
774 logfile = self.file_open(table._loggername, 'a')
775 logfile.write(message)
776 self.file_close(logfile)
777
778
779 - def __init__(self, db,uri,pool_size=0, folder=None, db_codec='UTF-8',
780 credential_decoder=IDENTITY, driver_args={},
781 adapter_args={},do_connect=True, after_connection=None):
782 self.db = db
783 self.dbengine = "None"
784 self.uri = uri
785 self.pool_size = pool_size
786 self.folder = folder
787 self.db_codec = db_codec
788 self._after_connection = after_connection
789 class Dummy(object):
790 lastrowid = 1
791 def __getattr__(self, value):
792 return lambda *a, **b: []
793 self.connection = Dummy()
794 self.cursor = Dummy()
795
797 return '%s_sequence' % tablename
798
800 return '%s_sequence' % tablename
801
804
805 - def create_table(self, table,
806 migrate=True,
807 fake_migrate=False,
808 polymodel=None):
809 db = table._db
810 fields = []
811
812 postcreation_fields = []
813 sql_fields = {}
814 sql_fields_aux = {}
815 TFK = {}
816 tablename = table._tablename
817 sortable = 0
818 types = self.types
819 for field in table:
820 sortable += 1
821 field_name = field.name
822 field_type = field.type
823 if isinstance(field_type,SQLCustomType):
824 ftype = field_type.native or field_type.type
825 elif field_type.startswith('reference'):
826 referenced = field_type[10:].strip()
827 if referenced == '.':
828 referenced = tablename
829 constraint_name = self.constraint_name(tablename, field_name)
830 if not '.' in referenced \
831 and referenced != tablename \
832 and hasattr(table,'_primarykey'):
833 ftype = types['integer']
834 else:
835 if hasattr(table,'_primarykey'):
836 rtablename,rfieldname = referenced.split('.')
837 rtable = db[rtablename]
838 rfield = rtable[rfieldname]
839
840 if rfieldname in rtable._primarykey or \
841 rfield.unique:
842 ftype = types[rfield.type[:9]] % \
843 dict(length=rfield.length)
844
845 if not rfield.unique and len(rtable._primarykey)>1:
846
847 if rtablename not in TFK:
848 TFK[rtablename] = {}
849 TFK[rtablename][rfieldname] = field_name
850 else:
851 ftype = ftype + \
852 types['reference FK'] % dict(
853 constraint_name = constraint_name,
854 foreign_key = '%s (%s)' % (rtablename,
855 rfieldname),
856 table_name = tablename,
857 field_name = field_name,
858 on_delete_action=field.ondelete)
859 else:
860
861 if referenced in db:
862 id_fieldname = db[referenced]._id.name
863 elif referenced == tablename:
864 id_fieldname = table._id.name
865 else:
866 id_fieldname = 'id'
867 ftype = types[field_type[:9]] % dict(
868 index_name = field_name+'__idx',
869 field_name = field_name,
870 constraint_name = constraint_name,
871 foreign_key = '%s (%s)' % (referenced,
872 id_fieldname),
873 on_delete_action=field.ondelete)
874 elif field_type.startswith('list:reference'):
875 ftype = types[field_type[:14]]
876 elif field_type.startswith('decimal'):
877 precision, scale = map(int,field_type[8:-1].split(','))
878 ftype = types[field_type[:7]] % \
879 dict(precision=precision,scale=scale)
880 elif field_type.startswith('geo'):
881 if not hasattr(self,'srid'):
882 raise RuntimeError('Adapter does not support geometry')
883 srid = self.srid
884 geotype, parms = field_type[:-1].split('(')
885 if not geotype in types:
886 raise SyntaxError(
887 'Field: unknown field type: %s for %s' \
888 % (field_type, field_name))
889 ftype = types[geotype]
890 if self.dbengine == 'postgres' and geotype == 'geometry':
891
892 dimension = 2
893 parms = parms.split(',')
894 if len(parms) == 3:
895 schema, srid, dimension = parms
896 elif len(parms) == 2:
897 schema, srid = parms
898 else:
899 schema = parms[0]
900 ftype = "SELECT AddGeometryColumn ('%%(schema)s', '%%(tablename)s', '%%(fieldname)s', %%(srid)s, '%s', %%(dimension)s);" % types[geotype]
901 ftype = ftype % dict(schema=schema,
902 tablename=tablename,
903 fieldname=field_name, srid=srid,
904 dimension=dimension)
905 postcreation_fields.append(ftype)
906 elif not field_type in types:
907 raise SyntaxError('Field: unknown field type: %s for %s' % \
908 (field_type, field_name))
909 else:
910 ftype = types[field_type]\
911 % dict(length=field.length)
912 if not field_type.startswith('id') and \
913 not field_type.startswith('reference'):
914 if field.notnull:
915 ftype += ' NOT NULL'
916 else:
917 ftype += self.ALLOW_NULL()
918 if field.unique:
919 ftype += ' UNIQUE'
920 if field.custom_qualifier:
921 ftype += ' %s' % field.custom_qualifier
922
923
924 sql_fields[field_name] = dict(
925 length=field.length,
926 unique=field.unique,
927 notnull=field.notnull,
928 sortable=sortable,
929 type=str(field_type),
930 sql=ftype)
931
932 if field.notnull and not field.default is None:
933
934
935
936
937
938
939 not_null = self.NOT_NULL(field.default, field_type)
940 ftype = ftype.replace('NOT NULL', not_null)
941 sql_fields_aux[field_name] = dict(sql=ftype)
942
943
944 if not (self.dbengine == 'postgres' and \
945 field_type.startswith('geom')):
946 fields.append('%s %s' % (field_name, ftype))
947 other = ';'
948
949
950 if self.dbengine == 'mysql':
951 if not hasattr(table, "_primarykey"):
952 fields.append('PRIMARY KEY(%s)' % table._id.name)
953 other = ' ENGINE=InnoDB CHARACTER SET utf8;'
954
955 fields = ',\n '.join(fields)
956 for rtablename in TFK:
957 rfields = TFK[rtablename]
958 pkeys = db[rtablename]._primarykey
959 fkeys = [ rfields[k] for k in pkeys ]
960 fields = fields + ',\n ' + \
961 types['reference TFK'] % dict(
962 table_name = tablename,
963 field_name=', '.join(fkeys),
964 foreign_table = rtablename,
965 foreign_key = ', '.join(pkeys),
966 on_delete_action = field.ondelete)
967
968 if getattr(table,'_primarykey',None):
969 query = "CREATE TABLE %s(\n %s,\n %s) %s" % \
970 (tablename, fields,
971 self.PRIMARY_KEY(', '.join(table._primarykey)),other)
972 else:
973 query = "CREATE TABLE %s(\n %s\n)%s" % \
974 (tablename, fields, other)
975
976 if self.uri.startswith('sqlite:///') \
977 or self.uri.startswith('spatialite:///'):
978 path_encoding = sys.getfilesystemencoding() \
979 or locale.getdefaultlocale()[1] or 'utf8'
980 dbpath = self.uri[9:self.uri.rfind('/')]\
981 .decode('utf8').encode(path_encoding)
982 else:
983 dbpath = self.folder
984
985 if not migrate:
986 return query
987 elif self.uri.startswith('sqlite:memory')\
988 or self.uri.startswith('spatialite:memory'):
989 table._dbt = None
990 elif isinstance(migrate, str):
991 table._dbt = pjoin(dbpath, migrate)
992 else:
993 table._dbt = pjoin(
994 dbpath, '%s_%s.table' % (table._db._uri_hash, tablename))
995
996 if not table._dbt or not self.file_exists(table._dbt):
997 if table._dbt:
998 self.log('timestamp: %s\n%s\n'
999 % (datetime.datetime.today().isoformat(),
1000 query), table)
1001 if not fake_migrate:
1002 self.create_sequence_and_triggers(query,table)
1003 table._db.commit()
1004
1005
1006 for query in postcreation_fields:
1007 self.execute(query)
1008 table._db.commit()
1009 if table._dbt:
1010 tfile = self.file_open(table._dbt, 'w')
1011 pickle.dump(sql_fields, tfile)
1012 self.file_close(tfile)
1013 if fake_migrate:
1014 self.log('faked!\n', table)
1015 else:
1016 self.log('success!\n', table)
1017 else:
1018 tfile = self.file_open(table._dbt, 'r')
1019 try:
1020 sql_fields_old = pickle.load(tfile)
1021 except EOFError:
1022 self.file_close(tfile)
1023 raise RuntimeError('File %s appears corrupted' % table._dbt)
1024 self.file_close(tfile)
1025 if sql_fields != sql_fields_old:
1026 self.migrate_table(table,
1027 sql_fields, sql_fields_old,
1028 sql_fields_aux, None,
1029 fake_migrate=fake_migrate)
1030 return query
1031
1032 - def migrate_table(
1033 self,
1034 table,
1035 sql_fields,
1036 sql_fields_old,
1037 sql_fields_aux,
1038 logfile,
1039 fake_migrate=False,
1040 ):
1041
1042
1043 db = table._db
1044 db._migrated.append(table._tablename)
1045 tablename = table._tablename
1046 def fix(item):
1047 k,v=item
1048 if not isinstance(v,dict):
1049 v=dict(type='unknown',sql=v)
1050 return k.lower(),v
1051
1052
1053 sql_fields = dict(map(fix,sql_fields.iteritems()))
1054 sql_fields_old = dict(map(fix,sql_fields_old.iteritems()))
1055 sql_fields_aux = dict(map(fix,sql_fields_aux.iteritems()))
1056 if db._debug:
1057 logging.debug('migrating %s to %s' % (sql_fields_old,sql_fields))
1058
1059 keys = sql_fields.keys()
1060 for key in sql_fields_old:
1061 if not key in keys:
1062 keys.append(key)
1063 new_add = self.concat_add(tablename)
1064
1065 metadata_change = False
1066 sql_fields_current = copy.copy(sql_fields_old)
1067 for key in keys:
1068 query = None
1069 if not key in sql_fields_old:
1070 sql_fields_current[key] = sql_fields[key]
1071 if self.dbengine in ('postgres',) and \
1072 sql_fields[key]['type'].startswith('geometry'):
1073
1074 query = [ sql_fields[key]['sql'] ]
1075 else:
1076 query = ['ALTER TABLE %s ADD %s %s;' % \
1077 (tablename, key,
1078 sql_fields_aux[key]['sql'].replace(', ', new_add))]
1079 metadata_change = True
1080 elif self.dbengine in ('sqlite', 'spatialite'):
1081 if key in sql_fields:
1082 sql_fields_current[key] = sql_fields[key]
1083 metadata_change = True
1084 elif not key in sql_fields:
1085 del sql_fields_current[key]
1086 ftype = sql_fields_old[key]['type']
1087 if (self.dbengine in ('postgres',) and
1088 ftype.startswith('geometry')):
1089 geotype, parms = ftype[:-1].split('(')
1090 schema = parms.split(',')[0]
1091 query = [ "SELECT DropGeometryColumn ('%(schema)s', "+
1092 "'%(table)s', '%(field)s');" %
1093 dict(schema=schema, table=tablename, field=key,) ]
1094 elif self.dbengine in ('firebird',):
1095 query = ['ALTER TABLE %s DROP %s;' % (tablename, key)]
1096 else:
1097 query = ['ALTER TABLE %s DROP COLUMN %s;' %
1098 (tablename, key)]
1099 metadata_change = True
1100 elif sql_fields[key]['sql'] != sql_fields_old[key]['sql'] \
1101 and not (key in table.fields and
1102 isinstance(table[key].type, SQLCustomType)) \
1103 and not sql_fields[key]['type'].startswith('reference')\
1104 and not sql_fields[key]['type'].startswith('double')\
1105 and not sql_fields[key]['type'].startswith('id'):
1106 sql_fields_current[key] = sql_fields[key]
1107 t = tablename
1108 tt = sql_fields_aux[key]['sql'].replace(', ', new_add)
1109 if self.dbengine in ('firebird',):
1110 drop_expr = 'ALTER TABLE %s DROP %s;'
1111 else:
1112 drop_expr = 'ALTER TABLE %s DROP COLUMN %s;'
1113 key_tmp = key + '__tmp'
1114 query = ['ALTER TABLE %s ADD %s %s;' % (t, key_tmp, tt),
1115 'UPDATE %s SET %s=%s;' % (t, key_tmp, key),
1116 drop_expr % (t, key),
1117 'ALTER TABLE %s ADD %s %s;' % (t, key, tt),
1118 'UPDATE %s SET %s=%s;' % (t, key, key_tmp),
1119 drop_expr % (t, key_tmp)]
1120 metadata_change = True
1121 elif sql_fields[key]['type'] != sql_fields_old[key]['type']:
1122 sql_fields_current[key] = sql_fields[key]
1123 metadata_change = True
1124
1125 if query:
1126 self.log('timestamp: %s\n'
1127 % datetime.datetime.today().isoformat(), table)
1128 db['_lastsql'] = '\n'.join(query)
1129 for sub_query in query:
1130 self.log(sub_query + '\n', table)
1131 if fake_migrate:
1132 if db._adapter.commit_on_alter_table:
1133 self.save_dbt(table,sql_fields_current)
1134 self.log('faked!\n', table)
1135 else:
1136 self.execute(sub_query)
1137
1138
1139
1140
1141
1142 if db._adapter.commit_on_alter_table:
1143 db.commit()
1144 self.save_dbt(table,sql_fields_current)
1145 self.log('success!\n', table)
1146
1147 elif metadata_change:
1148 self.save_dbt(table,sql_fields_current)
1149
1150 if metadata_change and not (query and db._adapter.commit_on_alter_table):
1151 db.commit()
1152 self.save_dbt(table,sql_fields_current)
1153 self.log('success!\n', table)
1154
1155 - def save_dbt(self,table, sql_fields_current):
1156 tfile = self.file_open(table._dbt, 'w')
1157 pickle.dump(sql_fields_current, tfile)
1158 self.file_close(tfile)
1159
1160 - def LOWER(self, first):
1162
1163 - def UPPER(self, first):
1165
1166 - def COUNT(self, first, distinct=None):
1167 return ('COUNT(%s)' if not distinct else 'COUNT(DISTINCT %s)') \
1168 % self.expand(first)
1169
1171 return "EXTRACT(%s FROM %s)" % (what, self.expand(first))
1172
1173 - def EPOCH(self, first):
1175
1178
1181
1184
1187
1190
1191 - def NOT_NULL(self, default, field_type):
1192 return 'NOT NULL DEFAULT %s' % self.represent(default,field_type)
1193
1195 expressions = [self.expand(first)]+[self.expand(e) for e in second]
1196 return 'COALESCE(%s)' % ','.join(expressions)
1197
1200
1201 - def RAW(self, first):
1203
1206
1208 return 'SUBSTR(%s,%s,%s)' % (self.expand(field), parameters[0], parameters[1])
1209
1211 return 'PRIMARY KEY(%s)' % key
1212
1213 - def _drop(self, table, mode):
1214 return ['DROP TABLE %s;' % table]
1215
1216 - def drop(self, table, mode=''):
1230
1231 - def _insert(self, table, fields):
1238
1240 return 'INSERT INTO %s DEFAULT VALUES;' % table
1241
1242 - def insert(self, table, fields):
1243 query = self._insert(table,fields)
1244 try:
1245 self.execute(query)
1246 except Exception:
1247 e = sys.exc_info()[1]
1248 if hasattr(table,'_on_insert_error'):
1249 return table._on_insert_error(table,fields,e)
1250 raise e
1251 if hasattr(table,'_primarykey'):
1252 return dict([(k[0].name, k[1]) for k in fields \
1253 if k[0].name in table._primarykey])
1254 id = self.lastrowid(table)
1255 if not isinstance(id,int):
1256 return id
1257 rid = Reference(id)
1258 (rid._table, rid._record) = (table, None)
1259 return rid
1260
1262 return [self.insert(table,item) for item in items]
1263
1264 - def NOT(self, first):
1266
1267 - def AND(self, first, second):
1269
1270 - def OR(self, first, second):
1272
1273 - def BELONGS(self, first, second):
1274 if isinstance(second, str):
1275 return '(%s IN (%s))' % (self.expand(first), second[:-1])
1276 if not second:
1277 return '(1=0)'
1278 items = ','.join(self.expand(item, first.type) for item in second)
1279 return '(%s IN (%s))' % (self.expand(first), items)
1280
1281 - def REGEXP(self, first, second):
1282 "regular expression operator"
1283 raise NotImplementedError
1284
1285 - def LIKE(self, first, second):
1286 "case sensitive like operator"
1287 raise NotImplementedError
1288
1289 - def ILIKE(self, first, second):
1290 "case in-sensitive like operator"
1291 return '(%s LIKE %s)' % (self.expand(first),
1292 self.expand(second, 'string'))
1293
1295 return '(%s LIKE %s)' % (self.expand(first),
1296 self.expand(second+'%', 'string'))
1297
1299 return '(%s LIKE %s)' % (self.expand(first),
1300 self.expand('%'+second, 'string'))
1301
1302 - def CONTAINS(self,first,second,case_sensitive=False):
1303 if first.type in ('string','text', 'json'):
1304 if isinstance(second,Expression):
1305 second = Expression(None,self.CONCAT('%',Expression(
1306 None,self.REPLACE(second,('%','%%'))),'%'))
1307 else:
1308 second = '%'+str(second).replace('%','%%')+'%'
1309 elif first.type.startswith('list:'):
1310 if isinstance(second,Expression):
1311 second = Expression(None,self.CONCAT(
1312 '%|',Expression(None,self.REPLACE(
1313 Expression(None,self.REPLACE(
1314 second,('%','%%'))),('|','||'))),'|%'))
1315 else:
1316 second = '%|'+str(second).replace('%','%%')\
1317 .replace('|','||')+'|%'
1318 op = case_sensitive and self.LIKE or self.ILIKE
1319 return op(first,second)
1320
1321 - def EQ(self, first, second=None):
1326
1327 - def NE(self, first, second=None):
1332
1333 - def LT(self,first,second=None):
1334 if second is None:
1335 raise RuntimeError("Cannot compare %s < None" % first)
1336 return '(%s < %s)' % (self.expand(first),
1337 self.expand(second,first.type))
1338
1339 - def LE(self,first,second=None):
1340 if second is None:
1341 raise RuntimeError("Cannot compare %s <= None" % first)
1342 return '(%s <= %s)' % (self.expand(first),
1343 self.expand(second,first.type))
1344
1345 - def GT(self,first,second=None):
1346 if second is None:
1347 raise RuntimeError("Cannot compare %s > None" % first)
1348 return '(%s > %s)' % (self.expand(first),
1349 self.expand(second,first.type))
1350
1351 - def GE(self,first,second=None):
1352 if second is None:
1353 raise RuntimeError("Cannot compare %s >= None" % first)
1354 return '(%s >= %s)' % (self.expand(first),
1355 self.expand(second,first.type))
1356
1358 return ftype in ('integer','boolean','double','bigint') or \
1359 ftype.startswith('decimal')
1360
1361 - def REPLACE(self, first, (second, third)):
1362 return 'REPLACE(%s,%s,%s)' % (self.expand(first,'string'),
1363 self.expand(second,'string'),
1364 self.expand(third,'string'))
1365
1368
1369 - def ADD(self, first, second):
1375
1376 - def SUB(self, first, second):
1379
1380 - def MUL(self, first, second):
1383
1384 - def DIV(self, first, second):
1387
1388 - def MOD(self, first, second):
1391
1392 - def AS(self, first, second):
1394
1395 - def ON(self, first, second):
1399
1402
1403 - def COMMA(self, first, second):
1405
1406 - def expand(self, expression, field_type=None):
1407 if isinstance(expression, Field):
1408 out = '%s.%s' % (expression.table._tablename, expression.name)
1409 if field_type == 'string' and not expression.type in (
1410 'string','text','json','password'):
1411 out = 'CAST(%s AS %s)' % (out, self.types['text'])
1412 return out
1413 elif isinstance(expression, (Expression, Query)):
1414 first = expression.first
1415 second = expression.second
1416 op = expression.op
1417 optional_args = expression.optional_args or {}
1418 if not second is None:
1419 out = op(first, second, **optional_args)
1420 elif not first is None:
1421 out = op(first,**optional_args)
1422 elif isinstance(op, str):
1423 if op.endswith(';'):
1424 op=op[:-1]
1425 out = '(%s)' % op
1426 else:
1427 out = op()
1428 return out
1429 elif field_type:
1430 return str(self.represent(expression,field_type))
1431 elif isinstance(expression,(list,tuple)):
1432 return ','.join(self.represent(item,field_type) \
1433 for item in expression)
1434 elif isinstance(expression, bool):
1435 return '1' if expression else '0'
1436 else:
1437 return str(expression)
1438
1441
1442 - def alias(self, table, alias):
1443 """
1444 Given a table object, makes a new table object
1445 with alias name.
1446 """
1447 other = copy.copy(table)
1448 other['_ot'] = other._ot or other._tablename
1449 other['ALL'] = SQLALL(other)
1450 other['_tablename'] = alias
1451 for fieldname in other.fields:
1452 other[fieldname] = copy.copy(other[fieldname])
1453 other[fieldname]._tablename = alias
1454 other[fieldname].tablename = alias
1455 other[fieldname].table = other
1456 table._db[alias] = other
1457 return other
1458
1460 tablename = table._tablename
1461 return ['TRUNCATE TABLE %s %s;' % (tablename, mode or '')]
1462
1464
1465 try:
1466 queries = table._db._adapter._truncate(table, mode)
1467 for query in queries:
1468 self.log(query + '\n', table)
1469 self.execute(query)
1470 table._db.commit()
1471 self.log('success!\n', table)
1472 finally:
1473 pass
1474
1475 - def _update(self, tablename, query, fields):
1476 if query:
1477 if use_common_filters(query):
1478 query = self.common_filter(query, [tablename])
1479 sql_w = ' WHERE ' + self.expand(query)
1480 else:
1481 sql_w = ''
1482 sql_v = ','.join(['%s=%s' % (field.name,
1483 self.expand(value, field.type)) \
1484 for (field, value) in fields])
1485 tablename = "%s" % self.db[tablename]
1486 return 'UPDATE %s SET %s%s;' % (tablename, sql_v, sql_w)
1487
1488 - def update(self, tablename, query, fields):
1489 sql = self._update(tablename, query, fields)
1490 try:
1491 self.execute(sql)
1492 except Exception:
1493 e = sys.exc_info()[1]
1494 table = self.db[tablename]
1495 if hasattr(table,'_on_update_error'):
1496 return table._on_update_error(table,query,fields,e)
1497 raise e
1498 try:
1499 return self.cursor.rowcount
1500 except:
1501 return None
1502
1503 - def _delete(self, tablename, query):
1504 if query:
1505 if use_common_filters(query):
1506 query = self.common_filter(query, [tablename])
1507 sql_w = ' WHERE ' + self.expand(query)
1508 else:
1509 sql_w = ''
1510 return 'DELETE FROM %s%s;' % (tablename, sql_w)
1511
1512 - def delete(self, tablename, query):
1513 sql = self._delete(tablename, query)
1514
1515 db = self.db
1516 table = db[tablename]
1517 if self.dbengine in ('sqlite', 'spatialite') and table._referenced_by:
1518 deleted = [x[table._id.name] for x in db(query).select(table._id)]
1519
1520 self.execute(sql)
1521 try:
1522 counter = self.cursor.rowcount
1523 except:
1524 counter = None
1525
1526 if self.dbengine in ('sqlite', 'spatialite') and counter:
1527 for field in table._referenced_by:
1528 if field.type=='reference '+table._tablename \
1529 and field.ondelete=='CASCADE':
1530 db(field.belongs(deleted)).delete()
1531
1532 return counter
1533
1535 tablenames = self.tables(query)
1536 if len(tablenames)==1:
1537 return tablenames[0]
1538 elif len(tablenames)<1:
1539 raise RuntimeError("No table selected")
1540 else:
1541 raise RuntimeError("Too many tables selected")
1542
1544 db = self.db
1545 new_fields = []
1546 append = new_fields.append
1547 for item in fields:
1548 if isinstance(item,SQLALL):
1549 new_fields += item._table
1550 elif isinstance(item,str):
1551 if REGEX_TABLE_DOT_FIELD.match(item):
1552 tablename,fieldname = item.split('.')
1553 append(db[tablename][fieldname])
1554 else:
1555 append(Expression(db,lambda item=item:item))
1556 else:
1557 append(item)
1558
1559 if not new_fields:
1560 for table in tablenames:
1561 for field in db[table]:
1562 append(field)
1563 return new_fields
1564
1565 - def _select(self, query, fields, attributes):
1566 tables = self.tables
1567 for key in set(attributes.keys())-SELECT_ARGS:
1568 raise SyntaxError('invalid select attribute: %s' % key)
1569 args_get = attributes.get
1570 tablenames = tables(query)
1571 tablenames_for_common_filters = tablenames
1572 for field in fields:
1573 if isinstance(field, basestring) \
1574 and REGEX_TABLE_DOT_FIELD.match(field):
1575 tn,fn = field.split('.')
1576 field = self.db[tn][fn]
1577 for tablename in tables(field):
1578 if not tablename in tablenames:
1579 tablenames.append(tablename)
1580
1581 if len(tablenames) < 1:
1582 raise SyntaxError('Set: no tables selected')
1583 self._colnames = map(self.expand, fields)
1584 def geoexpand(field):
1585 if isinstance(field.type,str) and field.type.startswith('geometry'):
1586 field = field.st_astext()
1587 return self.expand(field)
1588 sql_f = ', '.join(map(geoexpand, fields))
1589 sql_o = ''
1590 sql_s = ''
1591 left = args_get('left', False)
1592 inner_join = args_get('join', False)
1593 distinct = args_get('distinct', False)
1594 groupby = args_get('groupby', False)
1595 orderby = args_get('orderby', False)
1596 having = args_get('having', False)
1597 limitby = args_get('limitby', False)
1598 orderby_on_limitby = args_get('orderby_on_limitby', True)
1599 for_update = args_get('for_update', False)
1600 if self.can_select_for_update is False and for_update is True:
1601 raise SyntaxError('invalid select attribute: for_update')
1602 if distinct is True:
1603 sql_s += 'DISTINCT'
1604 elif distinct:
1605 sql_s += 'DISTINCT ON (%s)' % distinct
1606 if inner_join:
1607 icommand = self.JOIN()
1608 if not isinstance(inner_join, (tuple, list)):
1609 inner_join = [inner_join]
1610 ijoint = [t._tablename for t in inner_join
1611 if not isinstance(t,Expression)]
1612 ijoinon = [t for t in inner_join if isinstance(t, Expression)]
1613 itables_to_merge={}
1614 [itables_to_merge.update(
1615 dict.fromkeys(tables(t))) for t in ijoinon]
1616 ijoinont = [t.first._tablename for t in ijoinon]
1617 [itables_to_merge.pop(t) for t in ijoinont
1618 if t in itables_to_merge]
1619 iimportant_tablenames = ijoint + ijoinont + itables_to_merge.keys()
1620 iexcluded = [t for t in tablenames
1621 if not t in iimportant_tablenames]
1622 if left:
1623 join = attributes['left']
1624 command = self.LEFT_JOIN()
1625 if not isinstance(join, (tuple, list)):
1626 join = [join]
1627 joint = [t._tablename for t in join
1628 if not isinstance(t, Expression)]
1629 joinon = [t for t in join if isinstance(t, Expression)]
1630
1631 tables_to_merge={}
1632 [tables_to_merge.update(
1633 dict.fromkeys(tables(t))) for t in joinon]
1634 joinont = [t.first._tablename for t in joinon]
1635 [tables_to_merge.pop(t) for t in joinont if t in tables_to_merge]
1636 tablenames_for_common_filters = [t for t in tablenames
1637 if not t in joinont ]
1638 important_tablenames = joint + joinont + tables_to_merge.keys()
1639 excluded = [t for t in tablenames
1640 if not t in important_tablenames ]
1641 else:
1642 excluded = tablenames
1643
1644 if use_common_filters(query):
1645 query = self.common_filter(query,tablenames_for_common_filters)
1646 sql_w = ' WHERE ' + self.expand(query) if query else ''
1647
1648 if inner_join and not left:
1649 sql_t = ', '.join([self.table_alias(t) for t in iexcluded + \
1650 itables_to_merge.keys()])
1651 for t in ijoinon:
1652 sql_t += ' %s %s' % (icommand, t)
1653 elif not inner_join and left:
1654 sql_t = ', '.join([self.table_alias(t) for t in excluded + \
1655 tables_to_merge.keys()])
1656 if joint:
1657 sql_t += ' %s %s' % (command,
1658 ','.join([self.table_alias(t) for t in joint]))
1659 for t in joinon:
1660 sql_t += ' %s %s' % (command, t)
1661 elif inner_join and left:
1662 all_tables_in_query = set(important_tablenames + \
1663 iimportant_tablenames + \
1664 tablenames)
1665 tables_in_joinon = set(joinont + ijoinont)
1666 tables_not_in_joinon = \
1667 all_tables_in_query.difference(tables_in_joinon)
1668 sql_t = ','.join([self.table_alias(t) for t in tables_not_in_joinon])
1669 for t in ijoinon:
1670 sql_t += ' %s %s' % (icommand, t)
1671 if joint:
1672 sql_t += ' %s %s' % (command,
1673 ','.join([self.table_alias(t) for t in joint]))
1674 for t in joinon:
1675 sql_t += ' %s %s' % (command, t)
1676 else:
1677 sql_t = ', '.join(self.table_alias(t) for t in tablenames)
1678 if groupby:
1679 if isinstance(groupby, (list, tuple)):
1680 groupby = xorify(groupby)
1681 sql_o += ' GROUP BY %s' % self.expand(groupby)
1682 if having:
1683 sql_o += ' HAVING %s' % attributes['having']
1684 if orderby:
1685 if isinstance(orderby, (list, tuple)):
1686 orderby = xorify(orderby)
1687 if str(orderby) == '<random>':
1688 sql_o += ' ORDER BY %s' % self.RANDOM()
1689 else:
1690 sql_o += ' ORDER BY %s' % self.expand(orderby)
1691 if (limitby and not groupby and tablenames and orderby_on_limitby and not orderby):
1692 sql_o += ' ORDER BY %s' % ', '.join(['%s.%s'%(t,x) for t in tablenames for x in (hasattr(self.db[t],'_primarykey') and self.db[t]._primarykey or [self.db[t]._id.name])])
1693
1694 sql = self.select_limitby(sql_s, sql_f, sql_t, sql_w, sql_o, limitby)
1695 if for_update and self.can_select_for_update is True:
1696 sql = sql.rstrip(';') + ' FOR UPDATE;'
1697 return sql
1698
1699 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
1700 if limitby:
1701 (lmin, lmax) = limitby
1702 sql_o += ' LIMIT %i OFFSET %i' % (lmax - lmin, lmin)
1703 return 'SELECT %s %s FROM %s%s%s;' % \
1704 (sql_s, sql_f, sql_t, sql_w, sql_o)
1705
1707 return self.cursor.fetchall()
1708
1710 args_get = attributes.get
1711 cache = args_get('cache',None)
1712 if not cache:
1713 self.execute(sql)
1714 rows = self._fetchall()
1715 else:
1716 (cache_model, time_expire) = cache
1717 key = self.uri + '/' + sql + '/rows'
1718 if len(key)>200: key = hashlib_md5(key).hexdigest()
1719 def _select_aux2():
1720 self.execute(sql)
1721 return self._fetchall()
1722 rows = cache_model(key,_select_aux2,time_expire)
1723 if isinstance(rows,tuple):
1724 rows = list(rows)
1725 limitby = args_get('limitby', None) or (0,)
1726 rows = self.rowslice(rows,limitby[0],None)
1727 processor = args_get('processor',self.parse)
1728 cacheable = args_get('cacheable',False)
1729 return processor(rows,fields,self._colnames,cacheable=cacheable)
1730
1731 - def select(self, query, fields, attributes):
1732 """
1733 Always returns a Rows object, possibly empty.
1734 """
1735 sql = self._select(query, fields, attributes)
1736 cache = attributes.get('cache', None)
1737 if cache and attributes.get('cacheable',False):
1738 del attributes['cache']
1739 (cache_model, time_expire) = cache
1740 key = self.uri + '/' + sql
1741 if len(key)>200: key = hashlib_md5(key).hexdigest()
1742 args = (sql,fields,attributes)
1743 return cache_model(
1744 key,
1745 lambda self=self,args=args:self._select_aux(*args),
1746 time_expire)
1747 else:
1748 return self._select_aux(sql,fields,attributes)
1749
1750 - def _count(self, query, distinct=None):
1751 tablenames = self.tables(query)
1752 if query:
1753 if use_common_filters(query):
1754 query = self.common_filter(query, tablenames)
1755 sql_w = ' WHERE ' + self.expand(query)
1756 else:
1757 sql_w = ''
1758 sql_t = ','.join(self.table_alias(t) for t in tablenames)
1759 if distinct:
1760 if isinstance(distinct,(list, tuple)):
1761 distinct = xorify(distinct)
1762 sql_d = self.expand(distinct)
1763 return 'SELECT count(DISTINCT %s) FROM %s%s;' % \
1764 (sql_d, sql_t, sql_w)
1765 return 'SELECT count(*) FROM %s%s;' % (sql_t, sql_w)
1766
1767 - def count(self, query, distinct=None):
1768 self.execute(self._count(query, distinct))
1769 return self.cursor.fetchone()[0]
1770
1782
1786
1790
1796
1799
1802
1805
1808
1811
1813 return '%s_%s__constraint' % (table,fieldname)
1814
1817
1819 if not self.connection: return None
1820 command = a[0]
1821 if hasattr(self,'filter_sql_command'):
1822 command = self.filter_sql_command(command)
1823 if self.db._debug:
1824 LOGGER.debug('SQL: %s' % command)
1825 self.db._lastsql = command
1826 t0 = time.time()
1827 ret = self.cursor.execute(command, *a[1:], **b)
1828 self.db._timings.append((command,time.time()-t0))
1829 del self.db._timings[:-TIMINGSSIZE]
1830 return ret
1831
1834
1836 field_is_type = fieldtype.startswith
1837 if isinstance(obj, CALLABLETYPES):
1838 obj = obj()
1839 if isinstance(fieldtype, SQLCustomType):
1840 value = fieldtype.encoder(obj)
1841 if fieldtype.type in ('string','text', 'json'):
1842 return self.adapt(value)
1843 return value
1844 if isinstance(obj, (Expression, Field)):
1845 return str(obj)
1846 if field_is_type('list:'):
1847 if not obj:
1848 obj = []
1849 elif not isinstance(obj, (list, tuple)):
1850 obj = [obj]
1851 if field_is_type('list:string'):
1852 obj = map(str,obj)
1853 else:
1854 obj = map(int,[o for o in obj if o != ''])
1855
1856 if isinstance(obj, (list, tuple)) and (not fieldtype == "json"):
1857 obj = bar_encode(obj)
1858 if obj is None:
1859 return 'NULL'
1860 if obj == '' and not fieldtype[:2] in ['st', 'te', 'js', 'pa', 'up']:
1861 return 'NULL'
1862 r = self.represent_exceptions(obj, fieldtype)
1863 if not r is None:
1864 return r
1865 if fieldtype == 'boolean':
1866 if obj and not str(obj)[:1].upper() in '0F':
1867 return self.smart_adapt(self.TRUE)
1868 else:
1869 return self.smart_adapt(self.FALSE)
1870 if fieldtype == 'id' or fieldtype == 'integer':
1871 return str(long(obj))
1872 if field_is_type('decimal'):
1873 return str(obj)
1874 elif field_is_type('reference'):
1875 if fieldtype.find('.')>0:
1876 return repr(obj)
1877 elif isinstance(obj, (Row, Reference)):
1878 return str(obj['id'])
1879 return str(long(obj))
1880 elif fieldtype == 'double':
1881 return repr(float(obj))
1882 if isinstance(obj, unicode):
1883 obj = obj.encode(self.db_codec)
1884 if fieldtype == 'blob':
1885 obj = base64.b64encode(str(obj))
1886 elif fieldtype == 'date':
1887 if isinstance(obj, (datetime.date, datetime.datetime)):
1888 obj = obj.isoformat()[:10]
1889 else:
1890 obj = str(obj)
1891 elif fieldtype == 'datetime':
1892 if isinstance(obj, datetime.datetime):
1893 obj = obj.isoformat(self.T_SEP)[:19]
1894 elif isinstance(obj, datetime.date):
1895 obj = obj.isoformat()[:10]+self.T_SEP+'00:00:00'
1896 else:
1897 obj = str(obj)
1898 elif fieldtype == 'time':
1899 if isinstance(obj, datetime.time):
1900 obj = obj.isoformat()[:10]
1901 else:
1902 obj = str(obj)
1903 elif fieldtype == 'json':
1904 if not self.native_json:
1905 if have_serializers:
1906 obj = serializers.json(obj)
1907 elif simplejson:
1908 obj = simplejson.dumps(obj)
1909 else:
1910 raise RuntimeError("missing simplejson")
1911 if not isinstance(obj,bytes):
1912 obj = bytes(obj)
1913 try:
1914 obj.decode(self.db_codec)
1915 except:
1916 obj = obj.decode('latin1').encode(self.db_codec)
1917 return self.adapt(obj)
1918
1921
1924
1925 - def rowslice(self, rows, minimum=0, maximum=None):
1926 """
1927 By default this function does nothing;
1928 overload when db does not do slicing.
1929 """
1930 return rows
1931
1932 - def parse_value(self, value, field_type, blob_decode=True):
1933 if field_type != 'blob' and isinstance(value, str):
1934 try:
1935 value = value.decode(self.db._db_codec)
1936 except Exception:
1937 pass
1938 if isinstance(value, unicode):
1939 value = value.encode('utf-8')
1940 if isinstance(field_type, SQLCustomType):
1941 value = field_type.decoder(value)
1942 if not isinstance(field_type, str) or value is None:
1943 return value
1944 elif field_type in ('string', 'text', 'password', 'upload', 'dict'):
1945 return value
1946 elif field_type.startswith('geo'):
1947 return value
1948 elif field_type == 'blob' and not blob_decode:
1949 return value
1950 else:
1951 key = REGEX_TYPE.match(field_type).group(0)
1952 return self.parsemap[key](value,field_type)
1953
1955 referee = field_type[10:].strip()
1956 if not '.' in referee:
1957 value = Reference(value)
1958 value._table, value._record = self.db[referee], None
1959 return value
1960
1962 return value == self.TRUE or str(value)[:1].lower() == 't'
1963
1965 if isinstance(value, datetime.datetime):
1966 return value.date()
1967 if not isinstance(value, (datetime.date,datetime.datetime)):
1968 (y, m, d) = map(int, str(value)[:10].strip().split('-'))
1969 value = datetime.date(y, m, d)
1970 return value
1971
1973 if not isinstance(value, datetime.time):
1974 time_items = map(int,str(value)[:8].strip().split(':')[:3])
1975 if len(time_items) == 3:
1976 (h, mi, s) = time_items
1977 else:
1978 (h, mi, s) = time_items + [0]
1979 value = datetime.time(h, mi, s)
1980 return value
1981
1983 if not isinstance(value, datetime.datetime):
1984 value = str(value)
1985 date_part,time_part,timezone = value[:10],value[11:19],value[19:]
1986 if '+' in timezone:
1987 ms,tz = timezone.split('+')
1988 h,m = tz.split(':')
1989 dt = datetime.timedelta(seconds=3600*int(h)+60*int(m))
1990 elif '-' in timezone:
1991 ms,tz = timezone.split('-')
1992 h,m = tz.split(':')
1993 dt = -datetime.timedelta(seconds=3600*int(h)+60*int(m))
1994 else:
1995 dt = None
1996 (y, m, d) = map(int,date_part.split('-'))
1997 time_parts = time_part and time_part.split(':')[:3] or (0,0,0)
1998 while len(time_parts)<3: time_parts.append(0)
1999 time_items = map(int,time_parts)
2000 (h, mi, s) = time_items
2001 value = datetime.datetime(y, m, d, h, mi, s)
2002 if dt:
2003 value = value + dt
2004 return value
2005
2007 return base64.b64decode(str(value))
2008
2010 decimals = int(field_type[8:-1].split(',')[-1])
2011 if self.dbengine in ('sqlite', 'spatialite'):
2012 value = ('%.' + str(decimals) + 'f') % value
2013 if not isinstance(value, decimal.Decimal):
2014 value = decimal.Decimal(str(value))
2015 return value
2016
2021
2026
2031
2032 - def parse_id(self, value, field_type):
2034
2037
2040
2042 if not self.native_json:
2043 if not isinstance(value, basestring):
2044 raise RuntimeError('json data not a string')
2045 if isinstance(value, unicode):
2046 value = value.encode('utf-8')
2047 if have_serializers:
2048 value = serializers.loads_json(value)
2049 elif simplejson:
2050 value = simplejson.loads(value)
2051 else:
2052 raise RuntimeError("missing simplejson")
2053 return value
2054
2056 self.parsemap = {
2057 'id':self.parse_id,
2058 'integer':self.parse_integer,
2059 'bigint':self.parse_integer,
2060 'float':self.parse_double,
2061 'double':self.parse_double,
2062 'reference':self.parse_reference,
2063 'boolean':self.parse_boolean,
2064 'date':self.parse_date,
2065 'time':self.parse_time,
2066 'datetime':self.parse_datetime,
2067 'blob':self.parse_blob,
2068 'decimal':self.parse_decimal,
2069 'json':self.parse_json,
2070 'list:integer':self.parse_list_integers,
2071 'list:reference':self.parse_list_references,
2072 'list:string':self.parse_list_strings,
2073 }
2074
2075 - def parse(self, rows, fields, colnames, blob_decode=True,
2076 cacheable = False):
2077 db = self.db
2078 virtualtables = []
2079 new_rows = []
2080 tmps = []
2081 for colname in colnames:
2082 if not REGEX_TABLE_DOT_FIELD.match(colname):
2083 tmps.append(None)
2084 else:
2085 (tablename, _the_sep_, fieldname) = colname.partition('.')
2086 table = db[tablename]
2087 field = table[fieldname]
2088 ft = field.type
2089 tmps.append((tablename,fieldname,table,field,ft))
2090 for (i,row) in enumerate(rows):
2091 new_row = Row()
2092 for (j,colname) in enumerate(colnames):
2093 value = row[j]
2094 tmp = tmps[j]
2095 if tmp:
2096 (tablename,fieldname,table,field,ft) = tmp
2097 if tablename in new_row:
2098 colset = new_row[tablename]
2099 else:
2100 colset = new_row[tablename] = Row()
2101 if tablename not in virtualtables:
2102 virtualtables.append(tablename)
2103 value = self.parse_value(value,ft,blob_decode)
2104 if field.filter_out:
2105 value = field.filter_out(value)
2106 colset[fieldname] = value
2107
2108
2109 if ft=='id' and fieldname!='id' and \
2110 not 'id' in table.fields:
2111 colset['id'] = value
2112
2113 if ft == 'id' and not cacheable:
2114
2115
2116
2117 if isinstance(self, GoogleDatastoreAdapter):
2118 id = value.key().id_or_name()
2119 colset[fieldname] = id
2120 colset.gae_item = value
2121 else:
2122 id = value
2123 colset.update_record = RecordUpdater(colset,table,id)
2124 colset.delete_record = RecordDeleter(table,id)
2125 if table._db._lazy_tables:
2126 colset['__get_lazy_reference__'] = LazyReferenceGetter(table, id)
2127 for rfield in table._referenced_by:
2128 referee_link = db._referee_name and \
2129 db._referee_name % dict(
2130 table=rfield.tablename,field=rfield.name)
2131 if referee_link and not referee_link in colset:
2132 colset[referee_link] = LazySet(rfield,id)
2133 else:
2134 if not '_extra' in new_row:
2135 new_row['_extra'] = Row()
2136 new_row['_extra'][colname] = \
2137 self.parse_value(value,
2138 fields[j].type,blob_decode)
2139 new_column_name = \
2140 REGEX_SELECT_AS_PARSER.search(colname)
2141 if not new_column_name is None:
2142 column_name = new_column_name.groups(0)
2143 setattr(new_row,column_name[0],value)
2144 new_rows.append(new_row)
2145 rowsobj = Rows(db, new_rows, colnames, rawrows=rows)
2146
2147
2148 for tablename in virtualtables:
2149 table = db[tablename]
2150 fields_virtual = [(f,v) for (f,v) in table.iteritems()
2151 if isinstance(v,FieldVirtual)]
2152 fields_lazy = [(f,v) for (f,v) in table.iteritems()
2153 if isinstance(v,FieldMethod)]
2154 if fields_virtual or fields_lazy:
2155 for row in rowsobj.records:
2156 box = row[tablename]
2157 for f,v in fields_virtual:
2158 try:
2159 box[f] = v.f(row)
2160 except AttributeError:
2161 pass
2162 for f,v in fields_lazy:
2163 try:
2164 box[f] = (v.handler or VirtualCommand)(v.f,row)
2165 except AttributeError:
2166 pass
2167
2168
2169 for item in table.virtualfields:
2170 try:
2171 rowsobj = rowsobj.setvirtualfields(**{tablename:item})
2172 except (KeyError, AttributeError):
2173
2174 pass
2175 return rowsobj
2176
2178 tenant_fieldname = self.db._request_tenant
2179
2180 for tablename in tablenames:
2181 table = self.db[tablename]
2182
2183
2184 if table._common_filter != None:
2185 query = query & table._common_filter(query)
2186
2187
2188 if tenant_fieldname in table:
2189 default = table[tenant_fieldname].default
2190 if not default is None:
2191 newquery = table[tenant_fieldname] == default
2192 if query is None:
2193 query = newquery
2194 else:
2195 query = query & newquery
2196 return query
2197
2198 - def CASE(self,query,t,f):
2199 def represent(x):
2200 types = {type(True):'boolean',type(0):'integer',type(1.0):'double'}
2201 if x is None: return 'NULL'
2202 elif isinstance(x,Expression): return str(x)
2203 else: return self.represent(x,types.get(type(x),'string'))
2204 return Expression(self.db,'CASE WHEN %s THEN %s ELSE %s END' % \
2205 (self.expand(query),represent(t),represent(f)))
2206
2212 drivers = ('sqlite2','sqlite3')
2213
2214 can_select_for_update = None
2215
2217 return "web2py_extract('%s',%s)" % (what, self.expand(field))
2218
2219 @staticmethod
2221 table = {
2222 'year': (0, 4),
2223 'month': (5, 7),
2224 'day': (8, 10),
2225 'hour': (11, 13),
2226 'minute': (14, 16),
2227 'second': (17, 19),
2228 }
2229 try:
2230 if lookup != 'epoch':
2231 (i, j) = table[lookup]
2232 return int(s[i:j])
2233 else:
2234 return time.mktime(datetime.datetime.strptime(s, '%Y-%m-%d %H:%M:%S').timetuple())
2235 except:
2236 return None
2237
2238 @staticmethod
2240 return re.compile(expression).search(item) is not None
2241
2242 - def __init__(self, db, uri, pool_size=0, folder=None, db_codec ='UTF-8',
2243 credential_decoder=IDENTITY, driver_args={},
2244 adapter_args={}, do_connect=True, after_connection=None):
2245 self.db = db
2246 self.dbengine = "sqlite"
2247 self.uri = uri
2248 if do_connect: self.find_driver(adapter_args)
2249 self.pool_size = 0
2250 self.folder = folder
2251 self.db_codec = db_codec
2252 self._after_connection = after_connection
2253 self.find_or_make_work_folder()
2254 path_encoding = sys.getfilesystemencoding() \
2255 or locale.getdefaultlocale()[1] or 'utf8'
2256 if uri.startswith('sqlite:memory'):
2257 self.dbpath = ':memory:'
2258 else:
2259 self.dbpath = uri.split('://',1)[1]
2260 if self.dbpath[0] != '/':
2261 if PYTHON_VERSION == 2:
2262 self.dbpath = pjoin(
2263 self.folder.decode(path_encoding).encode('utf8'), self.dbpath)
2264 else:
2265 self.dbpath = pjoin(self.folder, self.dbpath)
2266 if not 'check_same_thread' in driver_args:
2267 driver_args['check_same_thread'] = False
2268 if not 'detect_types' in driver_args and do_connect:
2269 driver_args['detect_types'] = self.driver.PARSE_DECLTYPES
2270 def connector(dbpath=self.dbpath, driver_args=driver_args):
2271 return self.driver.Connection(dbpath, **driver_args)
2272 self.connector = connector
2273 if do_connect: self.reconnect()
2274
2280
2282 tablename = table._tablename
2283 return ['DELETE FROM %s;' % tablename,
2284 "DELETE FROM sqlite_sequence WHERE name='%s';" % tablename]
2285
2288
2289 - def REGEXP(self,first,second):
2290 return '(%s REGEXP %s)' % (self.expand(first),
2291 self.expand(second,'string'))
2292
2293 - def select(self, query, fields, attributes):
2294 """
2295 Simulate SELECT ... FOR UPDATE with BEGIN IMMEDIATE TRANSACTION.
2296 Note that the entire database, rather than one record, is locked
2297 (it will be locked eventually anyway by the following UPDATE).
2298 """
2299 if attributes.get('for_update', False) and not 'cache' in attributes:
2300 self.execute('BEGIN IMMEDIATE TRANSACTION;')
2301 return super(SQLiteAdapter, self).select(query, fields, attributes)
2302
2304 drivers = ('sqlite3','sqlite2')
2305
2306 types = copy.copy(BaseAdapter.types)
2307 types.update(geometry='GEOMETRY')
2308
2309 - def __init__(self, db, uri, pool_size=0, folder=None, db_codec ='UTF-8',
2310 credential_decoder=IDENTITY, driver_args={},
2311 adapter_args={}, do_connect=True, srid=4326, after_connection=None):
2312 self.db = db
2313 self.dbengine = "spatialite"
2314 self.uri = uri
2315 if do_connect: self.find_driver(adapter_args)
2316 self.pool_size = 0
2317 self.folder = folder
2318 self.db_codec = db_codec
2319 self._after_connection = after_connection
2320 self.find_or_make_work_folder()
2321 self.srid = srid
2322 path_encoding = sys.getfilesystemencoding() \
2323 or locale.getdefaultlocale()[1] or 'utf8'
2324 if uri.startswith('spatialite:memory'):
2325 self.dbpath = ':memory:'
2326 else:
2327 self.dbpath = uri.split('://',1)[1]
2328 if self.dbpath[0] != '/':
2329 self.dbpath = pjoin(
2330 self.folder.decode(path_encoding).encode('utf8'), self.dbpath)
2331 if not 'check_same_thread' in driver_args:
2332 driver_args['check_same_thread'] = False
2333 if not 'detect_types' in driver_args and do_connect:
2334 driver_args['detect_types'] = self.driver.PARSE_DECLTYPES
2335 def connector(dbpath=self.dbpath, driver_args=driver_args):
2336 return self.driver.Connection(dbpath, **driver_args)
2337 self.connector = connector
2338 if do_connect: self.reconnect()
2339
2352
2353
2354
2356 return 'AsGeoJSON(%s,%s,%s)' %(self.expand(first),
2357 second['precision'], second['options'])
2358
2359 - def ST_ASTEXT(self, first):
2360 return 'AsText(%s)' %(self.expand(first))
2361
2365
2369
2373
2377
2381
2383 return 'Simplify(%s,%s)' %(self.expand(first),
2384 self.expand(second, 'double'))
2385
2389
2393
2395 field_is_type = fieldtype.startswith
2396 if field_is_type('geo'):
2397 srid = 4326
2398 geotype, parms = fieldtype[:-1].split('(')
2399 parms = parms.split(',')
2400 if len(parms) >= 2:
2401 schema, srid = parms[:2]
2402
2403 value = "ST_GeomFromText('%s',%s)" %(obj, srid)
2404
2405
2406
2407
2408 return value
2409 return BaseAdapter.represent(self, obj, fieldtype)
2410
2413 drivers = ('zxJDBC_sqlite',)
2414
2415 - def __init__(self, db, uri, pool_size=0, folder=None, db_codec='UTF-8',
2416 credential_decoder=IDENTITY, driver_args={},
2417 adapter_args={}, do_connect=True, after_connection=None):
2440 self.connector = connector
2441 if do_connect: self.reconnect()
2442
2447
2450
2453 drivers = ('MySQLdb','pymysql', 'mysqlconnector')
2454
2455 commit_on_alter_table = True
2456 support_distributed_transaction = True
2457 types = {
2458 'boolean': 'CHAR(1)',
2459 'string': 'VARCHAR(%(length)s)',
2460 'text': 'LONGTEXT',
2461 'json': 'LONGTEXT',
2462 'password': 'VARCHAR(%(length)s)',
2463 'blob': 'LONGBLOB',
2464 'upload': 'VARCHAR(%(length)s)',
2465 'integer': 'INT',
2466 'bigint': 'BIGINT',
2467 'float': 'FLOAT',
2468 'double': 'DOUBLE',
2469 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
2470 'date': 'DATE',
2471 'time': 'TIME',
2472 'datetime': 'DATETIME',
2473 'id': 'INT AUTO_INCREMENT NOT NULL',
2474 'reference': 'INT, INDEX %(index_name)s (%(field_name)s), FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2475 'list:integer': 'LONGTEXT',
2476 'list:string': 'LONGTEXT',
2477 'list:reference': 'LONGTEXT',
2478 'big-id': 'BIGINT AUTO_INCREMENT NOT NULL',
2479 'big-reference': 'BIGINT, INDEX %(index_name)s (%(field_name)s), FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2480 }
2481
2482 QUOTE_TEMPLATE = "`%s`"
2483
2486
2489
2491 return 'SUBSTRING(%s,%s,%s)' % (self.expand(field),
2492 parameters[0], parameters[1])
2493
2494 - def EPOCH(self, first):
2496
2498 return 'CONCAT(%s)' % ','.join(self.expand(x,'string') for x in items)
2499
2500 - def REGEXP(self,first,second):
2501 return '(%s REGEXP %s)' % (self.expand(first),
2502 self.expand(second,'string'))
2503
2504 - def _drop(self,table,mode):
2505
2506 return ['SET FOREIGN_KEY_CHECKS=0;','DROP TABLE %s;' % table,
2507 'SET FOREIGN_KEY_CHECKS=1;']
2508
2510 return 'INSERT INTO %s VALUES (DEFAULT);' % table
2511
2514
2518
2521
2524
2525 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>[^?]+)(\?set_encoding=(?P<charset>\w+))?$')
2526
2527 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
2528 credential_decoder=IDENTITY, driver_args={},
2529 adapter_args={}, do_connect=True, after_connection=None):
2530 self.db = db
2531 self.dbengine = "mysql"
2532 self.uri = uri
2533 if do_connect: self.find_driver(adapter_args,uri)
2534 self.pool_size = pool_size
2535 self.folder = folder
2536 self.db_codec = db_codec
2537 self._after_connection = after_connection
2538 self.find_or_make_work_folder()
2539 ruri = uri.split('://',1)[1]
2540 m = self.REGEX_URI.match(ruri)
2541 if not m:
2542 raise SyntaxError(
2543 "Invalid URI string in DAL: %s" % self.uri)
2544 user = credential_decoder(m.group('user'))
2545 if not user:
2546 raise SyntaxError('User required')
2547 password = credential_decoder(m.group('password'))
2548 if not password:
2549 password = ''
2550 host = m.group('host')
2551 if not host:
2552 raise SyntaxError('Host name required')
2553 db = m.group('db')
2554 if not db:
2555 raise SyntaxError('Database name required')
2556 port = int(m.group('port') or '3306')
2557 charset = m.group('charset') or 'utf8'
2558 driver_args.update(db=db,
2559 user=credential_decoder(user),
2560 passwd=credential_decoder(password),
2561 host=host,
2562 port=port,
2563 charset=charset)
2564
2565
2566 def connector(driver_args=driver_args):
2567 return self.driver.connect(**driver_args)
2568 self.connector = connector
2569 if do_connect: self.reconnect()
2570
2572 self.execute('SET FOREIGN_KEY_CHECKS=1;')
2573 self.execute("SET sql_mode='NO_BACKSLASH_ESCAPES';")
2574
2576 self.execute('select last_insert_id();')
2577 return int(self.cursor.fetchone()[0])
2578
2579
2580 -class PostgreSQLAdapter(BaseAdapter):
2581 drivers = ('psycopg2','pg8000')
2582
2583 support_distributed_transaction = True
2584 types = {
2585 'boolean': 'CHAR(1)',
2586 'string': 'VARCHAR(%(length)s)',
2587 'text': 'TEXT',
2588 'json': 'TEXT',
2589 'password': 'VARCHAR(%(length)s)',
2590 'blob': 'BYTEA',
2591 'upload': 'VARCHAR(%(length)s)',
2592 'integer': 'INTEGER',
2593 'bigint': 'BIGINT',
2594 'float': 'FLOAT',
2595 'double': 'FLOAT8',
2596 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
2597 'date': 'DATE',
2598 'time': 'TIME',
2599 'datetime': 'TIMESTAMP',
2600 'id': 'SERIAL PRIMARY KEY',
2601 'reference': 'INTEGER REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2602 'list:integer': 'TEXT',
2603 'list:string': 'TEXT',
2604 'list:reference': 'TEXT',
2605 'geometry': 'GEOMETRY',
2606 'geography': 'GEOGRAPHY',
2607 'big-id': 'BIGSERIAL PRIMARY KEY',
2608 'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2609 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2610 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
2611
2612 }
2613
2614 QUOTE_TEMPLATE = '%s'
2615
2616 - def varquote(self,name):
2617 return varquote_aux(name,'"%s"')
2618
2619 - def adapt(self,obj):
2620 if self.driver_name == 'psycopg2':
2621 return psycopg2_adapt(obj).getquoted()
2622 elif self.driver_name == 'pg8000':
2623 return "'%s'" % str(obj).replace("%","%%").replace("'","''")
2624 else:
2625 return "'%s'" % str(obj).replace("'","''")
2626
2627 - def sequence_name(self,table):
2628 return '%s_id_Seq' % table
2629
2632
2633 - def ADD(self, first, second):
2634 t = first.type
2635 if t in ('text','string','password', 'json', 'upload','blob'):
2636 return '(%s || %s)' % (self.expand(first), self.expand(second, t))
2637 else:
2638 return '(%s + %s)' % (self.expand(first), self.expand(second, t))
2639
2642
2643 - def prepare(self,key):
2644 self.execute("PREPARE TRANSACTION '%s';" % key)
2645
2646 - def commit_prepared(self,key):
2647 self.execute("COMMIT PREPARED '%s';" % key)
2648
2649 - def rollback_prepared(self,key):
2650 self.execute("ROLLBACK PREPARED '%s';" % key)
2651
2652 - def create_sequence_and_triggers(self, query, table, **args):
2653
2654
2655
2656
2657 self.execute(query)
2658
2659 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:@]+)(\:(?P<port>[0-9]+))?/(?P<db>[^\?]+)(\?sslmode=(?P<sslmode>.+))?$')
2660
2661 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
2662 credential_decoder=IDENTITY, driver_args={},
2663 adapter_args={}, do_connect=True, srid=4326,
2664 after_connection=None):
2665 self.db = db
2666 self.dbengine = "postgres"
2667 self.uri = uri
2668 if do_connect: self.find_driver(adapter_args,uri)
2669 self.pool_size = pool_size
2670 self.folder = folder
2671 self.db_codec = db_codec
2672 self._after_connection = after_connection
2673 self.srid = srid
2674 self.find_or_make_work_folder()
2675 ruri = uri.split('://',1)[1]
2676 m = self.REGEX_URI.match(ruri)
2677 if not m:
2678 raise SyntaxError("Invalid URI string in DAL")
2679 user = credential_decoder(m.group('user'))
2680 if not user:
2681 raise SyntaxError('User required')
2682 password = credential_decoder(m.group('password'))
2683 if not password:
2684 password = ''
2685 host = m.group('host')
2686 if not host:
2687 raise SyntaxError('Host name required')
2688 db = m.group('db')
2689 if not db:
2690 raise SyntaxError('Database name required')
2691 port = m.group('port') or '5432'
2692 sslmode = m.group('sslmode')
2693 if sslmode:
2694 msg = ("dbname='%s' user='%s' host='%s' "
2695 "port=%s password='%s' sslmode='%s'") \
2696 % (db, user, host, port, password, sslmode)
2697 else:
2698 msg = ("dbname='%s' user='%s' host='%s' "
2699 "port=%s password='%s'") \
2700 % (db, user, host, port, password)
2701
2702 if self.driver:
2703 self.__version__ = "%s %s" % (self.driver.__name__,
2704 self.driver.__version__)
2705 else:
2706 self.__version__ = None
2707 def connector(msg=msg,driver_args=driver_args):
2708 return self.driver.connect(msg,**driver_args)
2709 self.connector = connector
2710 if do_connect: self.reconnect()
2711
2712 - def after_connection(self):
2713 self.connection.set_client_encoding('UTF8')
2714 self.execute("SET standard_conforming_strings=on;")
2715 self.try_json()
2716
2717 - def lastrowid(self,table):
2718 self.execute("select currval('%s')" % table._sequence_name)
2719 return int(self.cursor.fetchone()[0])
2720
2721 - def try_json(self):
2722
2723
2724 if self.driver_name == "pg8000":
2725 supports_json = self.connection.server_version >= "9.2.0"
2726 elif (self.driver_name == "psycopg2") and \
2727 (self.driver.__version__ >= "2.0.12"):
2728 supports_json = self.connection.server_version >= 90200
2729 elif self.driver_name == "zxJDBC":
2730 supports_json = self.connection.dbversion >= "9.2.0"
2731 else: supports_json = None
2732 if supports_json:
2733 self.types["json"] = "JSON"
2734 self.native_json = True
2735 else: LOGGER.debug("Your database version does not support the JSON data type (using TEXT instead)")
2736
2737 - def LIKE(self,first,second):
2738 args = (self.expand(first), self.expand(second,'string'))
2739 if not first.type in ('string', 'text', 'json'):
2740 return '(CAST(%s AS CHAR(%s)) LIKE %s)' % (args[0], first.length, args[1])
2741 else:
2742 return '(%s LIKE %s)' % args
2743
2744 - def ILIKE(self,first,second):
2745 args = (self.expand(first), self.expand(second,'string'))
2746 if not first.type in ('string', 'text', 'json'):
2747 return '(CAST(%s AS CHAR(%s)) LIKE %s)' % (args[0], first.length, args[1])
2748 else:
2749 return '(%s ILIKE %s)' % args
2750
2751 - def REGEXP(self,first,second):
2752 return '(%s ~ %s)' % (self.expand(first),
2753 self.expand(second,'string'))
2754
2755 - def STARTSWITH(self,first,second):
2756 return '(%s ILIKE %s)' % (self.expand(first),
2757 self.expand(second+'%','string'))
2758
2759 - def ENDSWITH(self,first,second):
2760 return '(%s ILIKE %s)' % (self.expand(first),
2761 self.expand('%'+second,'string'))
2762
2763
2764
2765 - def ST_ASGEOJSON(self, first, second):
2766 """
2767 http://postgis.org/docs/ST_AsGeoJSON.html
2768 """
2769 return 'ST_AsGeoJSON(%s,%s,%s,%s)' %(second['version'],
2770 self.expand(first), second['precision'], second['options'])
2771
2772 - def ST_ASTEXT(self, first):
2773 """
2774 http://postgis.org/docs/ST_AsText.html
2775 """
2776 return 'ST_AsText(%s)' %(self.expand(first))
2777
2778 - def ST_X(self, first):
2779 """
2780 http://postgis.org/docs/ST_X.html
2781 """
2782 return 'ST_X(%s)' %(self.expand(first))
2783
2784 - def ST_Y(self, first):
2785 """
2786 http://postgis.org/docs/ST_Y.html
2787 """
2788 return 'ST_Y(%s)' %(self.expand(first))
2789
2790 - def ST_CONTAINS(self, first, second):
2791 """
2792 http://postgis.org/docs/ST_Contains.html
2793 """
2794 return 'ST_Contains(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2795
2796 - def ST_DISTANCE(self, first, second):
2797 """
2798 http://postgis.org/docs/ST_Distance.html
2799 """
2800 return 'ST_Distance(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2801
2802 - def ST_EQUALS(self, first, second):
2803 """
2804 http://postgis.org/docs/ST_Equals.html
2805 """
2806 return 'ST_Equals(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2807
2808 - def ST_INTERSECTS(self, first, second):
2809 """
2810 http://postgis.org/docs/ST_Intersects.html
2811 """
2812 return 'ST_Intersects(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2813
2814 - def ST_OVERLAPS(self, first, second):
2815 """
2816 http://postgis.org/docs/ST_Overlaps.html
2817 """
2818 return 'ST_Overlaps(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2819
2820 - def ST_SIMPLIFY(self, first, second):
2821 """
2822 http://postgis.org/docs/ST_Simplify.html
2823 """
2824 return 'ST_Simplify(%s,%s)' %(self.expand(first), self.expand(second, 'double'))
2825
2826 - def ST_TOUCHES(self, first, second):
2827 """
2828 http://postgis.org/docs/ST_Touches.html
2829 """
2830 return 'ST_Touches(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2831
2832 - def ST_WITHIN(self, first, second):
2833 """
2834 http://postgis.org/docs/ST_Within.html
2835 """
2836 return 'ST_Within(%s,%s)' %(self.expand(first), self.expand(second, first.type))
2837
2838 - def represent(self, obj, fieldtype):
2839 field_is_type = fieldtype.startswith
2840 if field_is_type('geo'):
2841 srid = 4326
2842 geotype, parms = fieldtype[:-1].split('(')
2843 parms = parms.split(',')
2844 if len(parms) >= 2:
2845 schema, srid = parms[:2]
2846 if field_is_type('geometry'):
2847 value = "ST_GeomFromText('%s',%s)" %(obj, srid)
2848 elif field_is_type('geography'):
2849 value = "ST_GeogFromText('SRID=%s;%s')" %(srid, obj)
2850
2851
2852 return value
2853 return BaseAdapter.represent(self, obj, fieldtype)
2854
2855 -class NewPostgreSQLAdapter(PostgreSQLAdapter):
2856 drivers = ('psycopg2','pg8000')
2857
2858 types = {
2859 'boolean': 'CHAR(1)',
2860 'string': 'VARCHAR(%(length)s)',
2861 'text': 'TEXT',
2862 'json': 'TEXT',
2863 'password': 'VARCHAR(%(length)s)',
2864 'blob': 'BYTEA',
2865 'upload': 'VARCHAR(%(length)s)',
2866 'integer': 'INTEGER',
2867 'bigint': 'BIGINT',
2868 'float': 'FLOAT',
2869 'double': 'FLOAT8',
2870 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
2871 'date': 'DATE',
2872 'time': 'TIME',
2873 'datetime': 'TIMESTAMP',
2874 'id': 'SERIAL PRIMARY KEY',
2875 'reference': 'INTEGER REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2876 'list:integer': 'BIGINT[]',
2877 'list:string': 'TEXT[]',
2878 'list:reference': 'BIGINT[]',
2879 'geometry': 'GEOMETRY',
2880 'geography': 'GEOGRAPHY',
2881 'big-id': 'BIGSERIAL PRIMARY KEY',
2882 'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2883 }
2884
2885 - def parse_list_integers(self, value, field_type):
2887
2888 - def parse_list_references(self, value, field_type):
2889 return [self.parse_reference(r, field_type[5:]) for r in value]
2890
2891 - def parse_list_strings(self, value, field_type):
2893
2894 - def represent(self, obj, fieldtype):
2895 field_is_type = fieldtype.startswith
2896 if field_is_type('list:'):
2897 if not obj:
2898 obj = []
2899 elif not isinstance(obj, (list, tuple)):
2900 obj = [obj]
2901 if field_is_type('list:string'):
2902 obj = map(str,obj)
2903 else:
2904 obj = map(int,obj)
2905 return 'ARRAY[%s]' % ','.join(repr(item) for item in obj)
2906 return BaseAdapter.represent(self, obj, fieldtype)
2907
2908
2909 -class JDBCPostgreSQLAdapter(PostgreSQLAdapter):
2910 drivers = ('zxJDBC',)
2911
2912 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>.+)$')
2913
2914 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
2915 credential_decoder=IDENTITY, driver_args={},
2916 adapter_args={}, do_connect=True, after_connection=None ):
2917 self.db = db
2918 self.dbengine = "postgres"
2919 self.uri = uri
2920 if do_connect: self.find_driver(adapter_args,uri)
2921 self.pool_size = pool_size
2922 self.folder = folder
2923 self.db_codec = db_codec
2924 self._after_connection = after_connection
2925 self.find_or_make_work_folder()
2926 ruri = uri.split('://',1)[1]
2927 m = self.REGEX_URI.match(ruri)
2928 if not m:
2929 raise SyntaxError("Invalid URI string in DAL")
2930 user = credential_decoder(m.group('user'))
2931 if not user:
2932 raise SyntaxError('User required')
2933 password = credential_decoder(m.group('password'))
2934 if not password:
2935 password = ''
2936 host = m.group('host')
2937 if not host:
2938 raise SyntaxError('Host name required')
2939 db = m.group('db')
2940 if not db:
2941 raise SyntaxError('Database name required')
2942 port = m.group('port') or '5432'
2943 msg = ('jdbc:postgresql://%s:%s/%s' % (host, port, db), user, password)
2944 def connector(msg=msg,driver_args=driver_args):
2945 return self.driver.connect(*msg,**driver_args)
2946 self.connector = connector
2947 if do_connect: self.reconnect()
2948
2949 - def after_connection(self):
2950 self.connection.set_client_encoding('UTF8')
2951 self.execute('BEGIN;')
2952 self.execute("SET CLIENT_ENCODING TO 'UNICODE';")
2953 self.try_json()
2954
2957 drivers = ('cx_Oracle',)
2958
2959 commit_on_alter_table = False
2960 types = {
2961 'boolean': 'CHAR(1)',
2962 'string': 'VARCHAR2(%(length)s)',
2963 'text': 'CLOB',
2964 'json': 'CLOB',
2965 'password': 'VARCHAR2(%(length)s)',
2966 'blob': 'CLOB',
2967 'upload': 'VARCHAR2(%(length)s)',
2968 'integer': 'INT',
2969 'bigint': 'NUMBER',
2970 'float': 'FLOAT',
2971 'double': 'BINARY_DOUBLE',
2972 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
2973 'date': 'DATE',
2974 'time': 'CHAR(8)',
2975 'datetime': 'DATE',
2976 'id': 'NUMBER PRIMARY KEY',
2977 'reference': 'NUMBER, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2978 'list:integer': 'CLOB',
2979 'list:string': 'CLOB',
2980 'list:reference': 'CLOB',
2981 'big-id': 'NUMBER PRIMARY KEY',
2982 'big-reference': 'NUMBER, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2983 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
2984 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
2985 }
2986
2988 return '%s_sequence' % tablename
2989
2991 return '%s_trigger' % tablename
2992
2994 return 'LEFT OUTER JOIN'
2995
2997 return 'dbms_random.value'
2998
2999 - def NOT_NULL(self,default,field_type):
3000 return 'DEFAULT %s NOT NULL' % self.represent(default,field_type)
3001
3002 - def _drop(self,table,mode):
3005
3006 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3007 if limitby:
3008 (lmin, lmax) = limitby
3009 if len(sql_w) > 1:
3010 sql_w_row = sql_w + ' AND w_row > %i' % lmin
3011 else:
3012 sql_w_row = 'WHERE w_row > %i' % lmin
3013 return 'SELECT %s %s FROM (SELECT w_tmp.*, ROWNUM w_row FROM (SELECT %s FROM %s%s%s) w_tmp WHERE ROWNUM<=%i) %s %s %s;' % (sql_s, sql_f, sql_f, sql_t, sql_w, sql_o, lmax, sql_t, sql_w_row, sql_o)
3014 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
3015
3021
3023 if fieldtype == 'blob':
3024 obj = base64.b64encode(str(obj))
3025 return ":CLOB('%s')" % obj
3026 elif fieldtype == 'date':
3027 if isinstance(obj, (datetime.date, datetime.datetime)):
3028 obj = obj.isoformat()[:10]
3029 else:
3030 obj = str(obj)
3031 return "to_date('%s','yyyy-mm-dd')" % obj
3032 elif fieldtype == 'datetime':
3033 if isinstance(obj, datetime.datetime):
3034 obj = obj.isoformat()[:19].replace('T',' ')
3035 elif isinstance(obj, datetime.date):
3036 obj = obj.isoformat()[:10]+' 00:00:00'
3037 else:
3038 obj = str(obj)
3039 return "to_date('%s','yyyy-mm-dd hh24:mi:ss')" % obj
3040 return None
3041
3042 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
3043 credential_decoder=IDENTITY, driver_args={},
3044 adapter_args={}, do_connect=True, after_connection=None):
3045 self.db = db
3046 self.dbengine = "oracle"
3047 self.uri = uri
3048 if do_connect: self.find_driver(adapter_args,uri)
3049 self.pool_size = pool_size
3050 self.folder = folder
3051 self.db_codec = db_codec
3052 self._after_connection = after_connection
3053 self.find_or_make_work_folder()
3054 ruri = uri.split('://',1)[1]
3055 if not 'threaded' in driver_args:
3056 driver_args['threaded']=True
3057 def connector(uri=ruri,driver_args=driver_args):
3058 return self.driver.connect(uri,**driver_args)
3059 self.connector = connector
3060 if do_connect: self.reconnect()
3061
3063 self.execute("ALTER SESSION SET NLS_DATE_FORMAT = 'YYYY-MM-DD HH24:MI:SS';")
3064 self.execute("ALTER SESSION SET NLS_TIMESTAMP_FORMAT = 'YYYY-MM-DD HH24:MI:SS';")
3065
3066 oracle_fix = re.compile("[^']*('[^']*'[^']*)*\:(?P<clob>CLOB\('([^']+|'')*'\))")
3067
3068 - def execute(self, command, args=None):
3069 args = args or []
3070 i = 1
3071 while True:
3072 m = self.oracle_fix.match(command)
3073 if not m:
3074 break
3075 command = command[:m.start('clob')] + str(i) + command[m.end('clob'):]
3076 args.append(m.group('clob')[6:-2].replace("''", "'"))
3077 i += 1
3078 if command[-1:]==';':
3079 command = command[:-1]
3080 return self.log_execute(command, args)
3081
3083 tablename = table._tablename
3084 id_name = table._id.name
3085 sequence_name = table._sequence_name
3086 trigger_name = table._trigger_name
3087 self.execute(query)
3088 self.execute('CREATE SEQUENCE %s START WITH 1 INCREMENT BY 1 NOMAXVALUE MINVALUE -1;' % sequence_name)
3089 self.execute("""
3090 CREATE OR REPLACE TRIGGER %(trigger_name)s BEFORE INSERT ON %(tablename)s FOR EACH ROW
3091 DECLARE
3092 curr_val NUMBER;
3093 diff_val NUMBER;
3094 PRAGMA autonomous_transaction;
3095 BEGIN
3096 IF :NEW.%(id)s IS NOT NULL THEN
3097 EXECUTE IMMEDIATE 'SELECT %(sequence_name)s.nextval FROM dual' INTO curr_val;
3098 diff_val := :NEW.%(id)s - curr_val - 1;
3099 IF diff_val != 0 THEN
3100 EXECUTE IMMEDIATE 'alter sequence %(sequence_name)s increment by '|| diff_val;
3101 EXECUTE IMMEDIATE 'SELECT %(sequence_name)s.nextval FROM dual' INTO curr_val;
3102 EXECUTE IMMEDIATE 'alter sequence %(sequence_name)s increment by 1';
3103 END IF;
3104 END IF;
3105 SELECT %(sequence_name)s.nextval INTO :NEW.%(id)s FROM DUAL;
3106 END;
3107 """ % dict(trigger_name=trigger_name, tablename=tablename,
3108 sequence_name=sequence_name,id=id_name))
3109
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3125 if any(x[1]==cx_Oracle.CLOB for x in self.cursor.description):
3126 return [tuple([(c.read() if type(c) == cx_Oracle.LOB else c) \
3127 for c in r]) for r in self.cursor]
3128 else:
3129 return self.cursor.fetchall()
3130
3132 drivers = ('pyodbc',)
3133 T_SEP = 'T'
3134
3135 QUOTE_TEMPLATE = "[%s]"
3136
3137 types = {
3138 'boolean': 'BIT',
3139 'string': 'VARCHAR(%(length)s)',
3140 'text': 'TEXT',
3141 'json': 'TEXT',
3142 'password': 'VARCHAR(%(length)s)',
3143 'blob': 'IMAGE',
3144 'upload': 'VARCHAR(%(length)s)',
3145 'integer': 'INT',
3146 'bigint': 'BIGINT',
3147 'float': 'FLOAT',
3148 'double': 'FLOAT',
3149 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
3150 'date': 'DATETIME',
3151 'time': 'CHAR(8)',
3152 'datetime': 'DATETIME',
3153 'id': 'INT IDENTITY PRIMARY KEY',
3154 'reference': 'INT NULL, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3155 'list:integer': 'TEXT',
3156 'list:string': 'TEXT',
3157 'list:reference': 'TEXT',
3158 'geometry': 'geometry',
3159 'geography': 'geography',
3160 'big-id': 'BIGINT IDENTITY PRIMARY KEY',
3161 'big-reference': 'BIGINT NULL, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3162 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3163 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
3164 }
3165
3167 return '; ALTER TABLE %s ADD ' % tablename
3168
3171
3173 return "DATEPART(%s,%s)" % (what, self.expand(field))
3174
3176 return 'LEFT OUTER JOIN'
3177
3180
3183
3185 return 'SUBSTRING(%s,%s,%s)' % (self.expand(field), parameters[0], parameters[1])
3186
3188 return 'PRIMARY KEY CLUSTERED (%s)' % key
3189
3191 if what == 'LENGTH':
3192 what = 'LEN'
3193 return "%s(%s)" % (what, self.expand(first))
3194
3195
3196 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3197 if limitby:
3198 (lmin, lmax) = limitby
3199 sql_s += ' TOP %i' % lmax
3200 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
3201
3202 TRUE = 1
3203 FALSE = 0
3204
3205 REGEX_DSN = re.compile('^(?P<dsn>.+)$')
3206 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>[^\?]+)(\?(?P<urlargs>.*))?$')
3207 REGEX_ARGPATTERN = re.compile('(?P<argkey>[^=]+)=(?P<argvalue>[^&]*)')
3208
3209 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
3210 credential_decoder=IDENTITY, driver_args={},
3211 adapter_args={}, do_connect=True, srid=4326,
3212 after_connection=None):
3213 self.db = db
3214 self.dbengine = "mssql"
3215 self.uri = uri
3216 if do_connect: self.find_driver(adapter_args,uri)
3217 self.pool_size = pool_size
3218 self.folder = folder
3219 self.db_codec = db_codec
3220 self._after_connection = after_connection
3221 self.srid = srid
3222 self.find_or_make_work_folder()
3223
3224 ruri = uri.split('://',1)[1]
3225 if '@' not in ruri:
3226 try:
3227 m = self.REGEX_DSN.match(ruri)
3228 if not m:
3229 raise SyntaxError(
3230 'Parsing uri string(%s) has no result' % self.uri)
3231 dsn = m.group('dsn')
3232 if not dsn:
3233 raise SyntaxError('DSN required')
3234 except SyntaxError:
3235 e = sys.exc_info()[1]
3236 LOGGER.error('NdGpatch error')
3237 raise e
3238
3239 cnxn = dsn
3240 else:
3241 m = self.REGEX_URI.match(ruri)
3242 if not m:
3243 raise SyntaxError(
3244 "Invalid URI string in DAL: %s" % self.uri)
3245 user = credential_decoder(m.group('user'))
3246 if not user:
3247 raise SyntaxError('User required')
3248 password = credential_decoder(m.group('password'))
3249 if not password:
3250 password = ''
3251 host = m.group('host')
3252 if not host:
3253 raise SyntaxError('Host name required')
3254 db = m.group('db')
3255 if not db:
3256 raise SyntaxError('Database name required')
3257 port = m.group('port') or '1433'
3258
3259
3260
3261 argsdict = { 'DRIVER':'{SQL Server}' }
3262 urlargs = m.group('urlargs') or ''
3263 for argmatch in self.REGEX_ARGPATTERN.finditer(urlargs):
3264 argsdict[str(argmatch.group('argkey')).upper()] = argmatch.group('argvalue')
3265 urlargs = ';'.join(['%s=%s' % (ak, av) for (ak, av) in argsdict.iteritems()])
3266 cnxn = 'SERVER=%s;PORT=%s;DATABASE=%s;UID=%s;PWD=%s;%s' \
3267 % (host, port, db, user, password, urlargs)
3268 def connector(cnxn=cnxn,driver_args=driver_args):
3269 return self.driver.connect(cnxn,**driver_args)
3270 self.connector = connector
3271 if do_connect: self.reconnect()
3272
3274
3275 self.execute('SELECT SCOPE_IDENTITY();')
3276 return long(self.cursor.fetchone()[0])
3277
3278 - def rowslice(self,rows,minimum=0,maximum=None):
3279 if maximum is None:
3280 return rows[minimum:]
3281 return rows[minimum:maximum]
3282
3283 - def EPOCH(self, first):
3284 return "DATEDIFF(second, '1970-01-01 00:00:00', %s)" % self.expand(first)
3285
3288
3289
3290
3291
3292
3293 - def ST_ASTEXT(self, first):
3294 return '%s.STAsText()' %(self.expand(first))
3295
3298
3301
3304
3307
3310
3311
3312
3315
3318
3320 field_is_type = fieldtype.startswith
3321 if field_is_type('geometry'):
3322 srid = 0
3323 geotype, parms = fieldtype[:-1].split('(')
3324 if parms:
3325 srid = parms
3326 return "geometry::STGeomFromText('%s',%s)" %(obj, srid)
3327 elif fieldtype == 'geography':
3328 srid = 4326
3329 geotype, parms = fieldtype[:-1].split('(')
3330 if parms:
3331 srid = parms
3332 return "geography::STGeomFromText('%s',%s)" %(obj, srid)
3333
3334
3335 return "geometry::STGeomFromText('%s',%s)" %(obj, srid)
3336 return BaseAdapter.represent(self, obj, fieldtype)
3337
3340 """ experimental support for pagination in MSSQL"""
3341 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3342 if limitby:
3343 (lmin, lmax) = limitby
3344 if lmin == 0:
3345 sql_s += ' TOP %i' % lmax
3346 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
3347 lmin += 1
3348 sql_o_inner = sql_o[sql_o.find('ORDER BY ')+9:]
3349 sql_g_inner = sql_o[:sql_o.find('ORDER BY ')]
3350 sql_f_outer = ['f_%s' % f for f in range(len(sql_f.split(',')))]
3351 sql_f_inner = [f for f in sql_f.split(',')]
3352 sql_f_iproxy = ['%s AS %s' % (o, n) for (o, n) in zip(sql_f_inner, sql_f_outer)]
3353 sql_f_iproxy = ', '.join(sql_f_iproxy)
3354 sql_f_oproxy = ', '.join(sql_f_outer)
3355 return 'SELECT %s %s FROM (SELECT %s ROW_NUMBER() OVER (ORDER BY %s) AS w_row, %s FROM %s%s%s) TMP WHERE w_row BETWEEN %i AND %s;' % (sql_s,sql_f_oproxy,sql_s,sql_f,sql_f_iproxy,sql_t,sql_w,sql_g_inner,lmin,lmax)
3356 return 'SELECT %s %s FROM %s%s%s;' % (sql_s,sql_f,sql_t,sql_w,sql_o)
3357 - def rowslice(self,rows,minimum=0,maximum=None):
3359
3362 drivers = ('pyodbc',)
3363
3364 types = {
3365 'boolean': 'CHAR(1)',
3366 'string': 'NVARCHAR(%(length)s)',
3367 'text': 'NTEXT',
3368 'json': 'NTEXT',
3369 'password': 'NVARCHAR(%(length)s)',
3370 'blob': 'IMAGE',
3371 'upload': 'NVARCHAR(%(length)s)',
3372 'integer': 'INT',
3373 'bigint': 'BIGINT',
3374 'float': 'FLOAT',
3375 'double': 'FLOAT',
3376 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
3377 'date': 'DATETIME',
3378 'time': 'CHAR(8)',
3379 'datetime': 'DATETIME',
3380 'id': 'INT IDENTITY PRIMARY KEY',
3381 'reference': 'INT, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3382 'list:integer': 'NTEXT',
3383 'list:string': 'NTEXT',
3384 'list:reference': 'NTEXT',
3385 'big-id': 'BIGINT IDENTITY PRIMARY KEY',
3386 'big-reference': 'BIGINT, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3387 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3388 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
3389 }
3390
3392 value = BaseAdapter.represent(self, obj, fieldtype)
3393 if fieldtype in ('string','text', 'json') and value[:1]=="'":
3394 value = 'N'+value
3395 return value
3396
3399
3401 drivers = ('pyodbc',)
3402 T_SEP = ' '
3403
3404 types = {
3405 'boolean': 'BOOLEAN',
3406 'string': 'VARCHAR(%(length)s)',
3407 'text': 'BYTEA',
3408 'json': 'VARCHAR(%(length)s)',
3409 'password': 'VARCHAR(%(length)s)',
3410 'blob': 'BYTEA',
3411 'upload': 'VARCHAR(%(length)s)',
3412 'integer': 'INT',
3413 'bigint': 'BIGINT',
3414 'float': 'FLOAT',
3415 'double': 'DOUBLE PRECISION',
3416 'decimal': 'DECIMAL(%(precision)s,%(scale)s)',
3417 'date': 'DATE',
3418 'time': 'TIME',
3419 'datetime': 'DATETIME',
3420 'id': 'IDENTITY',
3421 'reference': 'INT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3422 'list:integer': 'BYTEA',
3423 'list:string': 'BYTEA',
3424 'list:reference': 'BYTEA',
3425 'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3426 }
3427
3428
3430 return "DATE_PART('%s', TIMESTAMP %s)" % (what, self.expand(first))
3431
3433 tablename = table._tablename
3434 return ['TRUNCATE %s %s;' % (tablename, mode or '')]
3435
3436 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3437 if limitby:
3438 (lmin, lmax) = limitby
3439 sql_o += ' LIMIT %i OFFSET %i' % (lmax - lmin, lmin)
3440 return 'SELECT %s %s FROM %s%s%s;' % \
3441 (sql_s, sql_f, sql_t, sql_w, sql_o)
3442
3444 self.execute('SELECT LAST_INSERT_ID();')
3445 return long(self.cursor.fetchone()[0])
3446
3449
3451 drivers = ('Sybase',)
3452
3453 types = {
3454 'boolean': 'BIT',
3455 'string': 'CHAR VARYING(%(length)s)',
3456 'text': 'TEXT',
3457 'json': 'TEXT',
3458 'password': 'CHAR VARYING(%(length)s)',
3459 'blob': 'IMAGE',
3460 'upload': 'CHAR VARYING(%(length)s)',
3461 'integer': 'INT',
3462 'bigint': 'BIGINT',
3463 'float': 'FLOAT',
3464 'double': 'FLOAT',
3465 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
3466 'date': 'DATETIME',
3467 'time': 'CHAR(8)',
3468 'datetime': 'DATETIME',
3469 'id': 'INT IDENTITY PRIMARY KEY',
3470 'reference': 'INT NULL, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3471 'list:integer': 'TEXT',
3472 'list:string': 'TEXT',
3473 'list:reference': 'TEXT',
3474 'geometry': 'geometry',
3475 'geography': 'geography',
3476 'big-id': 'BIGINT IDENTITY PRIMARY KEY',
3477 'big-reference': 'BIGINT NULL, CONSTRAINT %(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3478 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3479 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
3480 }
3481
3482
3483 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
3484 credential_decoder=IDENTITY, driver_args={},
3485 adapter_args={}, do_connect=True, srid=4326,
3486 after_connection=None):
3487 self.db = db
3488 self.dbengine = "sybase"
3489 self.uri = uri
3490 if do_connect: self.find_driver(adapter_args,uri)
3491 self.pool_size = pool_size
3492 self.folder = folder
3493 self.db_codec = db_codec
3494 self._after_connection = after_connection
3495 self.srid = srid
3496 self.find_or_make_work_folder()
3497
3498 ruri = uri.split('://',1)[1]
3499 if '@' not in ruri:
3500 try:
3501 m = self.REGEX_DSN.match(ruri)
3502 if not m:
3503 raise SyntaxError(
3504 'Parsing uri string(%s) has no result' % self.uri)
3505 dsn = m.group('dsn')
3506 if not dsn:
3507 raise SyntaxError('DSN required')
3508 except SyntaxError:
3509 e = sys.exc_info()[1]
3510 LOGGER.error('NdGpatch error')
3511 raise e
3512 else:
3513 m = self.REGEX_URI.match(uri)
3514 if not m:
3515 raise SyntaxError(
3516 "Invalid URI string in DAL: %s" % self.uri)
3517 user = credential_decoder(m.group('user'))
3518 if not user:
3519 raise SyntaxError('User required')
3520 password = credential_decoder(m.group('password'))
3521 if not password:
3522 password = ''
3523 host = m.group('host')
3524 if not host:
3525 raise SyntaxError('Host name required')
3526 db = m.group('db')
3527 if not db:
3528 raise SyntaxError('Database name required')
3529 port = m.group('port') or '1433'
3530
3531 dsn = 'sybase:host=%s:%s;dbname=%s' % (host,port,db)
3532
3533 driver_args.update(user = credential_decoder(user),
3534 password = credential_decoder(password))
3535
3536 def connector(dsn=dsn,driver_args=driver_args):
3537 return self.driver.connect(dsn,**driver_args)
3538 self.connector = connector
3539 if do_connect: self.reconnect()
3540
3543 drivers = ('kinterbasdb','firebirdsql','fdb','pyodbc')
3544
3545 commit_on_alter_table = False
3546 support_distributed_transaction = True
3547 types = {
3548 'boolean': 'CHAR(1)',
3549 'string': 'VARCHAR(%(length)s)',
3550 'text': 'BLOB SUB_TYPE 1',
3551 'json': 'BLOB SUB_TYPE 1',
3552 'password': 'VARCHAR(%(length)s)',
3553 'blob': 'BLOB SUB_TYPE 0',
3554 'upload': 'VARCHAR(%(length)s)',
3555 'integer': 'INTEGER',
3556 'bigint': 'BIGINT',
3557 'float': 'FLOAT',
3558 'double': 'DOUBLE PRECISION',
3559 'decimal': 'DECIMAL(%(precision)s,%(scale)s)',
3560 'date': 'DATE',
3561 'time': 'TIME',
3562 'datetime': 'TIMESTAMP',
3563 'id': 'INTEGER PRIMARY KEY',
3564 'reference': 'INTEGER REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3565 'list:integer': 'BLOB SUB_TYPE 1',
3566 'list:string': 'BLOB SUB_TYPE 1',
3567 'list:reference': 'BLOB SUB_TYPE 1',
3568 'big-id': 'BIGINT PRIMARY KEY',
3569 'big-reference': 'BIGINT REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3570 }
3571
3573 return 'genid_%s' % tablename
3574
3576 return 'trg_id_%s' % tablename
3577
3580
3581 - def EPOCH(self, first):
3582 return "DATEDIFF(second, '1970-01-01 00:00:00', %s)" % self.expand(first)
3583
3584 - def NOT_NULL(self,default,field_type):
3585 return 'DEFAULT %s NOT NULL' % self.represent(default,field_type)
3586
3588 return 'SUBSTRING(%s from %s for %s)' % (self.expand(field), parameters[0], parameters[1])
3589
3592
3593 - def CONTAINS(self,first,second,case_sensitive=False):
3599
3600 - def _drop(self,table,mode):
3603
3604 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3605 if limitby:
3606 (lmin, lmax) = limitby
3607 sql_s = ' FIRST %i SKIP %i %s' % (lmax - lmin, lmin, sql_s)
3608 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
3609
3611 return ['DELETE FROM %s;' % table._tablename,
3612 'SET GENERATOR %s TO 0;' % table._sequence_name]
3613
3614 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>.+?)(\?set_encoding=(?P<charset>\w+))?$')
3615
3616 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
3617 credential_decoder=IDENTITY, driver_args={},
3618 adapter_args={}, do_connect=True, after_connection=None):
3619 self.db = db
3620 self.dbengine = "firebird"
3621 self.uri = uri
3622 if do_connect: self.find_driver(adapter_args,uri)
3623 self.pool_size = pool_size
3624 self.folder = folder
3625 self.db_codec = db_codec
3626 self._after_connection = after_connection
3627 self.find_or_make_work_folder()
3628 ruri = uri.split('://',1)[1]
3629 m = self.REGEX_URI.match(ruri)
3630 if not m:
3631 raise SyntaxError("Invalid URI string in DAL: %s" % self.uri)
3632 user = credential_decoder(m.group('user'))
3633 if not user:
3634 raise SyntaxError('User required')
3635 password = credential_decoder(m.group('password'))
3636 if not password:
3637 password = ''
3638 host = m.group('host')
3639 if not host:
3640 raise SyntaxError('Host name required')
3641 port = int(m.group('port') or 3050)
3642 db = m.group('db')
3643 if not db:
3644 raise SyntaxError('Database name required')
3645 charset = m.group('charset') or 'UTF8'
3646 driver_args.update(dsn='%s/%s:%s' % (host,port,db),
3647 user = credential_decoder(user),
3648 password = credential_decoder(password),
3649 charset = charset)
3650
3651 def connector(driver_args=driver_args):
3652 return self.driver.connect(**driver_args)
3653 self.connector = connector
3654 if do_connect: self.reconnect()
3655
3664
3669
3672 drivers = ('kinterbasdb','firebirdsql','fdb','pyodbc')
3673
3674 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<path>[^\?]+)(\?set_encoding=(?P<charset>\w+))?$')
3675
3676 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
3677 credential_decoder=IDENTITY, driver_args={},
3678 adapter_args={}, do_connect=True, after_connection=None):
3679 self.db = db
3680 self.dbengine = "firebird"
3681 self.uri = uri
3682 if do_connect: self.find_driver(adapter_args,uri)
3683 self.pool_size = pool_size
3684 self.folder = folder
3685 self.db_codec = db_codec
3686 self._after_connection = after_connection
3687 self.find_or_make_work_folder()
3688 ruri = uri.split('://',1)[1]
3689 m = self.REGEX_URI.match(ruri)
3690 if not m:
3691 raise SyntaxError(
3692 "Invalid URI string in DAL: %s" % self.uri)
3693 user = credential_decoder(m.group('user'))
3694 if not user:
3695 raise SyntaxError('User required')
3696 password = credential_decoder(m.group('password'))
3697 if not password:
3698 password = ''
3699 pathdb = m.group('path')
3700 if not pathdb:
3701 raise SyntaxError('Path required')
3702 charset = m.group('charset')
3703 if not charset:
3704 charset = 'UTF8'
3705 host = ''
3706 driver_args.update(host=host,
3707 database=pathdb,
3708 user=credential_decoder(user),
3709 password=credential_decoder(password),
3710 charset=charset)
3711
3712 def connector(driver_args=driver_args):
3713 return self.driver.connect(**driver_args)
3714 self.connector = connector
3715 if do_connect: self.reconnect()
3716
3822
3827
3830
3842
3844 drivers = ('pyodbc',)
3845
3846 types = {
3847 'boolean': 'CHAR(1)',
3848 'string': 'VARCHAR(%(length)s)',
3849 'text': 'CLOB',
3850 'json': 'CLOB',
3851 'password': 'VARCHAR(%(length)s)',
3852 'blob': 'BLOB',
3853 'upload': 'VARCHAR(%(length)s)',
3854 'integer': 'INT',
3855 'bigint': 'BIGINT',
3856 'float': 'REAL',
3857 'double': 'DOUBLE',
3858 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
3859 'date': 'DATE',
3860 'time': 'TIME',
3861 'datetime': 'TIMESTAMP',
3862 'id': 'INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL',
3863 'reference': 'INT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3864 'list:integer': 'CLOB',
3865 'list:string': 'CLOB',
3866 'list:reference': 'CLOB',
3867 'big-id': 'BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL',
3868 'big-reference': 'BIGINT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3869 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
3870 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
3871 }
3872
3874 return 'LEFT OUTER JOIN'
3875
3878
3879 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3880 if limitby:
3881 (lmin, lmax) = limitby
3882 sql_o += ' FETCH FIRST %i ROWS ONLY' % lmax
3883 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
3884
3886 if fieldtype == 'blob':
3887 obj = base64.b64encode(str(obj))
3888 return "BLOB('%s')" % obj
3889 elif fieldtype == 'datetime':
3890 if isinstance(obj, datetime.datetime):
3891 obj = obj.isoformat()[:19].replace('T','-').replace(':','.')
3892 elif isinstance(obj, datetime.date):
3893 obj = obj.isoformat()[:10]+'-00.00.00'
3894 return "'%s'" % obj
3895 return None
3896
3897 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
3898 credential_decoder=IDENTITY, driver_args={},
3899 adapter_args={}, do_connect=True, after_connection=None):
3912 self.connector = connector
3913 if do_connect: self.reconnect()
3914
3916 if command[-1:]==';':
3917 command = command[:-1]
3918 return self.log_execute(command)
3919
3921 self.execute('SELECT DISTINCT IDENTITY_VAL_LOCAL() FROM %s;' % table)
3922 return long(self.cursor.fetchone()[0])
3923
3924 - def rowslice(self,rows,minimum=0,maximum=None):
3925 if maximum is None:
3926 return rows[minimum:]
3927 return rows[minimum:maximum]
3928
3931 drivers = ('pyodbc',)
3932
3933 types = {
3934 'boolean': 'CHAR(1)',
3935 'string': 'VARCHAR(%(length)s)',
3936 'text': 'CLOB',
3937 'json': 'CLOB',
3938 'password': 'VARCHAR(%(length)s)',
3939 'blob': 'BLOB',
3940 'upload': 'VARCHAR(%(length)s)',
3941 'integer': 'INT',
3942 'bigint': 'BIGINT',
3943 'float': 'REAL',
3944 'double': 'DOUBLE',
3945 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
3946 'date': 'DATE',
3947 'time': 'TIME',
3948 'datetime': 'TIMESTAMP',
3949
3950
3951 'id': 'INT GENERATED ALWAYS AS IDENTITY',
3952 'reference': 'INT',
3953 'list:integer': 'CLOB',
3954 'list:string': 'CLOB',
3955 'list:reference': 'CLOB',
3956 'big-id': 'BIGINT GENERATED ALWAYS AS IDENTITY',
3957 'big-reference': 'BIGINT',
3958 'reference FK': ' REFERENCES %(foreign_key)s',
3959 'reference TFK': ' FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s)',
3960 }
3961
3962 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
3963 credential_decoder=IDENTITY, driver_args={},
3964 adapter_args={}, do_connect=True, after_connection=None):
3977 self.connector = connector
3978 if do_connect: self.reconnect()
3979
3981 return 'LEFT OUTER JOIN'
3982
3983
3984 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
3985 if limitby:
3986 (lmin, lmax) = limitby
3987 sql_s += ' TOP %i' % lmax
3988 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
3989
3991 tablename = table._tablename
3992 return ['DELETE FROM %s ALL;' % (tablename)]
3993
3994 INGRES_SEQNAME='ii***lineitemsequence'
3999 drivers = ('pyodbc',)
4000
4001 types = {
4002 'boolean': 'CHAR(1)',
4003 'string': 'VARCHAR(%(length)s)',
4004 'text': 'CLOB',
4005 'json': 'CLOB',
4006 'password': 'VARCHAR(%(length)s)',
4007 'blob': 'BLOB',
4008 'upload': 'VARCHAR(%(length)s)',
4009 'integer': 'INTEGER4',
4010 'bigint': 'BIGINT',
4011 'float': 'FLOAT',
4012 'double': 'FLOAT8',
4013 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
4014 'date': 'ANSIDATE',
4015 'time': 'TIME WITHOUT TIME ZONE',
4016 'datetime': 'TIMESTAMP WITHOUT TIME ZONE',
4017 'id': 'int not null unique with default next value for %s' % INGRES_SEQNAME,
4018 'reference': 'INT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
4019 'list:integer': 'CLOB',
4020 'list:string': 'CLOB',
4021 'list:reference': 'CLOB',
4022 'big-id': 'bigint not null unique with default next value for %s' % INGRES_SEQNAME,
4023 'big-reference': 'BIGINT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
4024 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
4025 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
4026 }
4027
4029 return 'LEFT OUTER JOIN'
4030
4033
4034 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
4035 if limitby:
4036 (lmin, lmax) = limitby
4037 fetch_amt = lmax - lmin
4038 if fetch_amt:
4039 sql_s += ' FIRST %d ' % (fetch_amt, )
4040 if lmin:
4041
4042 sql_o += ' OFFSET %d' % (lmin, )
4043 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
4044
4045 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
4046 credential_decoder=IDENTITY, driver_args={},
4047 adapter_args={}, do_connect=True, after_connection=None):
4048 self.db = db
4049 self.dbengine = "ingres"
4050 self._driver = pyodbc
4051 self.uri = uri
4052 if do_connect: self.find_driver(adapter_args,uri)
4053 self.pool_size = pool_size
4054 self.folder = folder
4055 self.db_codec = db_codec
4056 self._after_connection = after_connection
4057 self.find_or_make_work_folder()
4058 connstr = uri.split(':', 1)[1]
4059
4060 connstr = connstr.lstrip()
4061 while connstr.startswith('/'):
4062 connstr = connstr[1:]
4063 if '=' in connstr:
4064
4065 ruri = connstr
4066 else:
4067
4068 database_name = connstr
4069 default_driver_name = 'Ingres'
4070 vnode = '(local)'
4071 servertype = 'ingres'
4072 ruri = 'Driver={%s};Server=%s;Database=%s' % (default_driver_name, vnode, database_name)
4073 def connector(cnxn=ruri,driver_args=driver_args):
4074 return self.driver.connect(cnxn,**driver_args)
4075
4076 self.connector = connector
4077
4078
4079 if do_connect: self.reconnect()
4080
4082
4083
4084
4085 if hasattr(table,'_primarykey'):
4086 modify_tbl_sql = 'modify %s to btree unique on %s' % \
4087 (table._tablename,
4088 ', '.join(["'%s'" % x for x in table.primarykey]))
4089 self.execute(modify_tbl_sql)
4090 else:
4091 tmp_seqname='%s_iisq' % table._tablename
4092 query=query.replace(INGRES_SEQNAME, tmp_seqname)
4093 self.execute('create sequence %s' % tmp_seqname)
4094 self.execute(query)
4095 self.execute('modify %s to btree unique on %s' % (table._tablename, 'id'))
4096
4097
4099 tmp_seqname='%s_iisq' % table
4100 self.execute('select current value for %s' % tmp_seqname)
4101 return long(self.cursor.fetchone()[0])
4102
4105
4106 drivers = ('pyodbc',)
4107
4108 types = {
4109 'boolean': 'CHAR(1)',
4110 'string': 'NVARCHAR(%(length)s)',
4111 'text': 'NCLOB',
4112 'json': 'NCLOB',
4113 'password': 'NVARCHAR(%(length)s)',
4114 'blob': 'BLOB',
4115 'upload': 'VARCHAR(%(length)s)',
4116 'integer': 'INTEGER4',
4117 'bigint': 'BIGINT',
4118 'float': 'FLOAT',
4119 'double': 'FLOAT8',
4120 'decimal': 'NUMERIC(%(precision)s,%(scale)s)',
4121 'date': 'ANSIDATE',
4122 'time': 'TIME WITHOUT TIME ZONE',
4123 'datetime': 'TIMESTAMP WITHOUT TIME ZONE',
4124 'id': 'INTEGER4 not null unique with default next value for %s'% INGRES_SEQNAME,
4125 'reference': 'INTEGER4, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
4126 'list:integer': 'NCLOB',
4127 'list:string': 'NCLOB',
4128 'list:reference': 'NCLOB',
4129 'big-id': 'BIGINT not null unique with default next value for %s'% INGRES_SEQNAME,
4130 'big-reference': 'BIGINT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
4131 'reference FK': ', CONSTRAINT FK_%(constraint_name)s FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
4132 'reference TFK': ' CONSTRAINT FK_%(foreign_table)s_PK FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_table)s (%(foreign_key)s) ON DELETE %(on_delete_action)s',
4133 }
4134
4136 drivers = ('sapdb',)
4137
4138 support_distributed_transaction = False
4139 types = {
4140 'boolean': 'CHAR(1)',
4141 'string': 'VARCHAR(%(length)s)',
4142 'text': 'LONG',
4143 'json': 'LONG',
4144 'password': 'VARCHAR(%(length)s)',
4145 'blob': 'LONG',
4146 'upload': 'VARCHAR(%(length)s)',
4147 'integer': 'INT',
4148 'bigint': 'BIGINT',
4149 'float': 'FLOAT',
4150 'double': 'DOUBLE PRECISION',
4151 'decimal': 'FIXED(%(precision)s,%(scale)s)',
4152 'date': 'DATE',
4153 'time': 'TIME',
4154 'datetime': 'TIMESTAMP',
4155 'id': 'INT PRIMARY KEY',
4156 'reference': 'INT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
4157 'list:integer': 'LONG',
4158 'list:string': 'LONG',
4159 'list:reference': 'LONG',
4160 'big-id': 'BIGINT PRIMARY KEY',
4161 'big-reference': 'BIGINT, FOREIGN KEY (%(field_name)s) REFERENCES %(foreign_key)s ON DELETE %(on_delete_action)s',
4162 }
4163
4165 return '%s_id_Seq' % table
4166
4167 - def select_limitby(self, sql_s, sql_f, sql_t, sql_w, sql_o, limitby):
4168 if limitby:
4169 (lmin, lmax) = limitby
4170 if len(sql_w) > 1:
4171 sql_w_row = sql_w + ' AND w_row > %i' % lmin
4172 else:
4173 sql_w_row = 'WHERE w_row > %i' % lmin
4174 return '%s %s FROM (SELECT w_tmp.*, ROWNO w_row FROM (SELECT %s FROM %s%s%s) w_tmp WHERE ROWNO=%i) %s %s %s;' % (sql_s, sql_f, sql_f, sql_t, sql_w, sql_o, lmax, sql_t, sql_w_row, sql_o)
4175 return 'SELECT %s %s FROM %s%s%s;' % (sql_s, sql_f, sql_t, sql_w, sql_o)
4176
4178
4179 self.execute('CREATE SEQUENCE %s;' % table._sequence_name)
4180 self.execute("ALTER TABLE %s ALTER COLUMN %s SET DEFAULT NEXTVAL('%s');" \
4181 % (table._tablename, table._id.name, table._sequence_name))
4182 self.execute(query)
4183
4184 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:@]+)(\:(?P<port>[0-9]+))?/(?P<db>[^\?]+)(\?sslmode=(?P<sslmode>.+))?$')
4185
4186
4187 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
4188 credential_decoder=IDENTITY, driver_args={},
4189 adapter_args={}, do_connect=True, after_connection=None):
4190 self.db = db
4191 self.dbengine = "sapdb"
4192 self.uri = uri
4193 if do_connect: self.find_driver(adapter_args,uri)
4194 self.pool_size = pool_size
4195 self.folder = folder
4196 self.db_codec = db_codec
4197 self._after_connection = after_connection
4198 self.find_or_make_work_folder()
4199 ruri = uri.split('://',1)[1]
4200 m = self.REGEX_URI.match(ruri)
4201 if not m:
4202 raise SyntaxError("Invalid URI string in DAL")
4203 user = credential_decoder(m.group('user'))
4204 if not user:
4205 raise SyntaxError('User required')
4206 password = credential_decoder(m.group('password'))
4207 if not password:
4208 password = ''
4209 host = m.group('host')
4210 if not host:
4211 raise SyntaxError('Host name required')
4212 db = m.group('db')
4213 if not db:
4214 raise SyntaxError('Database name required')
4215 def connector(user=user, password=password, database=db,
4216 host=host, driver_args=driver_args):
4217 return self.driver.Connection(user, password, database,
4218 host, **driver_args)
4219 self.connector = connector
4220 if do_connect: self.reconnect()
4221
4223 self.execute("select %s.NEXTVAL from dual" % table._sequence_name)
4224 return long(self.cursor.fetchone()[0])
4225
4227 drivers = ('cubriddb',)
4228
4229 REGEX_URI = re.compile('^(?P<user>[^:@]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:/]+)(\:(?P<port>[0-9]+))?/(?P<db>[^?]+)(\?set_encoding=(?P<charset>\w+))?$')
4230
4231 - def __init__(self, db, uri, pool_size=0, folder=None, db_codec='UTF-8',
4232 credential_decoder=IDENTITY, driver_args={},
4233 adapter_args={}, do_connect=True, after_connection=None):
4234 self.db = db
4235 self.dbengine = "cubrid"
4236 self.uri = uri
4237 if do_connect: self.find_driver(adapter_args,uri)
4238 self.pool_size = pool_size
4239 self.folder = folder
4240 self.db_codec = db_codec
4241 self._after_connection = after_connection
4242 self.find_or_make_work_folder()
4243 ruri = uri.split('://',1)[1]
4244 m = self.REGEX_URI.match(ruri)
4245 if not m:
4246 raise SyntaxError(
4247 "Invalid URI string in DAL: %s" % self.uri)
4248 user = credential_decoder(m.group('user'))
4249 if not user:
4250 raise SyntaxError('User required')
4251 password = credential_decoder(m.group('password'))
4252 if not password:
4253 password = ''
4254 host = m.group('host')
4255 if not host:
4256 raise SyntaxError('Host name required')
4257 db = m.group('db')
4258 if not db:
4259 raise SyntaxError('Database name required')
4260 port = int(m.group('port') or '30000')
4261 charset = m.group('charset') or 'utf8'
4262 user = credential_decoder(user)
4263 passwd = credential_decoder(password)
4264 def connector(host=host,port=port,db=db,
4265 user=user,passwd=password,driver_args=driver_args):
4266 return self.driver.connect(host,port,db,user,passwd,**driver_args)
4267 self.connector = connector
4268 if do_connect: self.reconnect()
4269
4271 self.execute('SET FOREIGN_KEY_CHECKS=1;')
4272 self.execute("SET sql_mode='NO_BACKSLASH_ESCAPES';")
4273
4278
4279 web2py_filesystem = False
4280
4282 return self.db._adapter.escape(obj)
4283
4285 if not db._adapter.dbengine in ('mysql', 'postgres', 'sqlite'):
4286 raise RuntimeError("only MySQL/Postgres/SQLite can store metadata .table files in database for now")
4287 self.db = db
4288 self.filename = filename
4289 self.mode = mode
4290 if not self.web2py_filesystem:
4291 if db._adapter.dbengine == 'mysql':
4292 sql = "CREATE TABLE IF NOT EXISTS web2py_filesystem (path VARCHAR(255), content LONGTEXT, PRIMARY KEY(path) ) ENGINE=InnoDB;"
4293 elif db._adapter.dbengine in ('postgres', 'sqlite'):
4294 sql = "CREATE TABLE IF NOT EXISTS web2py_filesystem (path VARCHAR(255), content TEXT, PRIMARY KEY(path));"
4295 self.db.executesql(sql)
4296 DatabaseStoredFile.web2py_filesystem = True
4297 self.p=0
4298 self.data = ''
4299 if mode in ('r','rw','a'):
4300 query = "SELECT content FROM web2py_filesystem WHERE path='%s'" \
4301 % filename
4302 rows = self.db.executesql(query)
4303 if rows:
4304 self.data = rows[0][0]
4305 elif exists(filename):
4306 datafile = open(filename, 'r')
4307 try:
4308 self.data = datafile.read()
4309 finally:
4310 datafile.close()
4311 elif mode in ('r','rw'):
4312 raise RuntimeError("File %s does not exist" % filename)
4313
4314 - def read(self, bytes):
4315 data = self.data[self.p:self.p+bytes]
4316 self.p += len(data)
4317 return data
4318
4320 i = self.data.find('\n',self.p)+1
4321 if i>0:
4322 data, self.p = self.data[self.p:i], i
4323 else:
4324 data, self.p = self.data[self.p:], len(self.data)
4325 return data
4326
4329
4331 if self.db is not None:
4332 self.db.executesql(
4333 "DELETE FROM web2py_filesystem WHERE path='%s'" % self.filename)
4334 query = "INSERT INTO web2py_filesystem(path,content) VALUES ('%s','%s')"\
4335 % (self.filename, self.data.replace("'","''"))
4336 self.db.executesql(query)
4337 self.db.commit()
4338 self.db = None
4339
4342
4343 @staticmethod
4345 if exists(filename):
4346 return True
4347 query = "SELECT path FROM web2py_filesystem WHERE path='%s'" % filename
4348 try:
4349 if db.executesql(query):
4350 return True
4351 except Exception, e:
4352 if not (db._adapter.isOperationalError(e) or
4353 db._adapter.isProgrammingError(e)):
4354 raise
4355
4356 tb = traceback.format_exc()
4357 LOGGER.error("Could not retrieve %s\n%s" % (filename, tb))
4358 return False
4359
4362
4365
4366 - def file_open(self, filename, mode='rb', lock=True):
4368
4371
4373 query = "DELETE FROM web2py_filesystem WHERE path='%s'" % filename
4374 self.db.executesql(query)
4375 self.db.commit()
4376
4378 uploads_in_blob = True
4379
4380 REGEX_URI = re.compile('^(?P<instance>.*)/(?P<db>.*)$')
4381
4382 - def __init__(self, db, uri='google:sql://realm:domain/database',
4383 pool_size=0, folder=None, db_codec='UTF-8',
4384 credential_decoder=IDENTITY, driver_args={},
4385 adapter_args={}, do_connect=True, after_connection=None):
4386
4387 self.db = db
4388 self.dbengine = "mysql"
4389 self.uri = uri
4390 self.pool_size = pool_size
4391 self.db_codec = db_codec
4392 self._after_connection = after_connection
4393 if do_connect: self.find_driver(adapter_args, uri)
4394 self.folder = folder or pjoin('$HOME',THREAD_LOCAL.folder.split(
4395 os.sep+'applications'+os.sep,1)[1])
4396 ruri = uri.split("://")[1]
4397 m = self.REGEX_URI.match(ruri)
4398 if not m:
4399 raise SyntaxError("Invalid URI string in SQLDB: %s" % self.uri)
4400 instance = credential_decoder(m.group('instance'))
4401 self.dbstring = db = credential_decoder(m.group('db'))
4402 driver_args['instance'] = instance
4403 if not 'charset' in driver_args:
4404 driver_args['charset'] = 'utf8'
4405 self.createdb = createdb = adapter_args.get('createdb',True)
4406 if not createdb:
4407 driver_args['database'] = db
4408 def connector(driver_args=driver_args):
4409 return rdbms.connect(**driver_args)
4410 self.connector = connector
4411 if do_connect: self.reconnect()
4412
4414 if self.createdb:
4415
4416 self.execute('CREATE DATABASE IF NOT EXISTS %s' % self.dbstring)
4417 self.execute('USE %s' % self.dbstring)
4418 self.execute("SET FOREIGN_KEY_CHECKS=1;")
4419 self.execute("SET sql_mode='NO_BACKSLASH_ESCAPES';")
4420
4421 - def execute(self, command, *a, **b):
4423
4425 self.adapter_args = adapter_args
4426 self.driver = "google"
4427
4429 can_select_for_update = False
4430
4431 @staticmethod
4433 if isinstance(obj, str):
4434 return obj.decode('utf8')
4435 elif not isinstance(obj, unicode):
4436 return unicode(obj)
4437 return obj
4438
4440 return table._id > 0
4441
4443 field_is_type = fieldtype.startswith
4444 if isinstance(obj, CALLABLETYPES):
4445 obj = obj()
4446 if isinstance(fieldtype, SQLCustomType):
4447 return fieldtype.encoder(obj)
4448 if isinstance(obj, (Expression, Field)):
4449 raise SyntaxError("non supported on GAE")
4450 if self.dbengine == 'google:datastore':
4451 if isinstance(fieldtype, gae.Property):
4452 return obj
4453 is_string = isinstance(fieldtype,str)
4454 is_list = is_string and field_is_type('list:')
4455 if is_list:
4456 if not obj:
4457 obj = []
4458 if not isinstance(obj, (list, tuple)):
4459 obj = [obj]
4460 if obj == '' and not \
4461 (is_string and fieldtype[:2] in ['st','te', 'pa','up']):
4462 return None
4463 if not obj is None:
4464 if isinstance(obj, list) and not is_list:
4465 obj = [self.represent(o, fieldtype) for o in obj]
4466 elif fieldtype in ('integer','bigint','id'):
4467 obj = long(obj)
4468 elif fieldtype == 'double':
4469 obj = float(obj)
4470 elif is_string and field_is_type('reference'):
4471 if isinstance(obj, (Row, Reference)):
4472 obj = obj['id']
4473 obj = long(obj)
4474 elif fieldtype == 'boolean':
4475 if obj and not str(obj)[0].upper() in '0F':
4476 obj = True
4477 else:
4478 obj = False
4479 elif fieldtype == 'date':
4480 if not isinstance(obj, datetime.date):
4481 (y, m, d) = map(int,str(obj).strip().split('-'))
4482 obj = datetime.date(y, m, d)
4483 elif isinstance(obj,datetime.datetime):
4484 (y, m, d) = (obj.year, obj.month, obj.day)
4485 obj = datetime.date(y, m, d)
4486 elif fieldtype == 'time':
4487 if not isinstance(obj, datetime.time):
4488 time_items = map(int,str(obj).strip().split(':')[:3])
4489 if len(time_items) == 3:
4490 (h, mi, s) = time_items
4491 else:
4492 (h, mi, s) = time_items + [0]
4493 obj = datetime.time(h, mi, s)
4494 elif fieldtype == 'datetime':
4495 if not isinstance(obj, datetime.datetime):
4496 (y, m, d) = map(int,str(obj)[:10].strip().split('-'))
4497 time_items = map(int,str(obj)[11:].strip().split(':')[:3])
4498 while len(time_items)<3:
4499 time_items.append(0)
4500 (h, mi, s) = time_items
4501 obj = datetime.datetime(y, m, d, h, mi, s)
4502 elif fieldtype == 'blob':
4503 pass
4504 elif fieldtype == 'json':
4505 if isinstance(obj, basestring):
4506 obj = self.to_unicode(obj)
4507 if have_serializers:
4508 obj = serializers.loads_json(obj)
4509 elif simplejson:
4510 obj = simplejson.loads(obj)
4511 else:
4512 raise RuntimeError("missing simplejson")
4513 elif is_string and field_is_type('list:string'):
4514 return map(self.to_unicode,obj)
4515 elif is_list:
4516 return map(int,obj)
4517 else:
4518 obj = self.to_unicode(obj)
4519 return obj
4520
4522 return 'insert %s in %s' % (fields, table)
4523
4524 - def _count(self,query,distinct=None):
4525 return 'count %s' % repr(query)
4526
4527 - def _select(self,query,fields,attributes):
4528 return 'select %s where %s' % (repr(fields), repr(query))
4529
4530 - def _delete(self,tablename, query):
4531 return 'delete %s where %s' % (repr(tablename),repr(query))
4532
4533 - def _update(self,tablename,query,fields):
4534 return 'update %s (%s) where %s' % (repr(tablename),
4535 repr(fields),repr(query))
4536
4538 """
4539 remember: no transactions on many NoSQL
4540 """
4541 pass
4542
4544 """
4545 remember: no transactions on many NoSQL
4546 """
4547 pass
4548
4550 """
4551 remember: no transactions on many NoSQL
4552 """
4553 pass
4554
4555
4556
4557 - def OR(self,first,second): raise SyntaxError("Not supported")
4558 - def AND(self,first,second): raise SyntaxError("Not supported")
4559 - def AS(self,first,second): raise SyntaxError("Not supported")
4560 - def ON(self,first,second): raise SyntaxError("Not supported")
4561 - def STARTSWITH(self,first,second=None): raise SyntaxError("Not supported")
4562 - def ENDSWITH(self,first,second=None): raise SyntaxError("Not supported")
4563 - def ADD(self,first,second): raise SyntaxError("Not supported")
4564 - def SUB(self,first,second): raise SyntaxError("Not supported")
4565 - def MUL(self,first,second): raise SyntaxError("Not supported")
4566 - def DIV(self,first,second): raise SyntaxError("Not supported")
4567 - def LOWER(self,first): raise SyntaxError("Not supported")
4568 - def UPPER(self,first): raise SyntaxError("Not supported")
4570 - def LENGTH(self, first): raise SyntaxError("Not supported")
4571 - def AGGREGATE(self,first,what): raise SyntaxError("Not supported")
4572 - def LEFT_JOIN(self): raise SyntaxError("Not supported")
4573 - def RANDOM(self): raise SyntaxError("Not supported")
4574 - def SUBSTRING(self,field,parameters): raise SyntaxError("Not supported")
4575 - def PRIMARY_KEY(self,key): raise SyntaxError("Not supported")
4576 - def ILIKE(self,first,second): raise SyntaxError("Not supported")
4577 - def drop(self,table,mode): raise SyntaxError("Not supported")
4578 - def alias(self,table,alias): raise SyntaxError("Not supported")
4579 - def migrate_table(self,*a,**b): raise SyntaxError("Not supported")
4581 - def prepare(self,key): raise SyntaxError("Not supported")
4584 - def concat_add(self,table): raise SyntaxError("Not supported")
4585 - def constraint_name(self, table, fieldname): raise SyntaxError("Not supported")
4587 - def log_execute(self,*a,**b): raise SyntaxError("Not supported")
4588 - def execute(self,*a,**b): raise SyntaxError("Not supported")
4590 - def lastrowid(self,table): raise SyntaxError("Not supported")
4591 - def rowslice(self,rows,minimum=0,maximum=None): raise SyntaxError("Not supported")
4592
4593
4594 -class GAEF(object):
4595 - def __init__(self,name,op,value,apply):
4596 self.name=name=='id' and '__key__' or name
4597 self.op=op
4598 self.value=value
4599 self.apply=apply
4601 return '(%s %s %s:%s)' % (self.name, self.op, repr(self.value), type(self.value))
4602
4604 uploads_in_blob = True
4605 types = {}
4606
4608 - def file_open(self, filename, mode='rb', lock=True): pass
4610
4611 REGEX_NAMESPACE = re.compile('.*://(?P<namespace>.+)')
4612
4613 - def __init__(self,db,uri,pool_size=0,folder=None,db_codec ='UTF-8',
4614 credential_decoder=IDENTITY, driver_args={},
4615 adapter_args={}, do_connect=True, after_connection=None):
4616 self.types.update({
4617 'boolean': gae.BooleanProperty,
4618 'string': (lambda **kwargs: gae.StringProperty(multiline=True, **kwargs)),
4619 'text': gae.TextProperty,
4620 'json': gae.TextProperty,
4621 'password': gae.StringProperty,
4622 'blob': gae.BlobProperty,
4623 'upload': gae.StringProperty,
4624 'integer': gae.IntegerProperty,
4625 'bigint': gae.IntegerProperty,
4626 'float': gae.FloatProperty,
4627 'double': gae.FloatProperty,
4628 'decimal': GAEDecimalProperty,
4629 'date': gae.DateProperty,
4630 'time': gae.TimeProperty,
4631 'datetime': gae.DateTimeProperty,
4632 'id': None,
4633 'reference': gae.IntegerProperty,
4634 'list:string': (lambda **kwargs: gae.StringListProperty(default=None, **kwargs)),
4635 'list:integer': (lambda **kwargs: gae.ListProperty(int,default=None, **kwargs)),
4636 'list:reference': (lambda **kwargs: gae.ListProperty(int,default=None, **kwargs)),
4637 })
4638 self.db = db
4639 self.uri = uri
4640 self.dbengine = 'google:datastore'
4641 self.folder = folder
4642 db['_lastsql'] = ''
4643 self.db_codec = 'UTF-8'
4644 self._after_connection = after_connection
4645 self.pool_size = 0
4646 match = self.REGEX_NAMESPACE.match(uri)
4647 if match:
4648 namespace_manager.set_namespace(match.group('namespace'))
4649
4650 - def parse_id(self, value, field_type):
4652
4653 - def create_table(self,table,migrate=True,fake_migrate=False, polymodel=None):
4654 myfields = {}
4655 for field in table:
4656 if isinstance(polymodel,Table) and field.name in polymodel.fields():
4657 continue
4658 attr = {}
4659 if isinstance(field.custom_qualifier, dict):
4660
4661 attr = field.custom_qualifier
4662 field_type = field.type
4663 if isinstance(field_type, SQLCustomType):
4664 ftype = self.types[field_type.native or field_type.type](**attr)
4665 elif isinstance(field_type, gae.Property):
4666 ftype = field_type
4667 elif field_type.startswith('id'):
4668 continue
4669 elif field_type.startswith('decimal'):
4670 precision, scale = field_type[7:].strip('()').split(',')
4671 precision = int(precision)
4672 scale = int(scale)
4673 ftype = GAEDecimalProperty(precision, scale, **attr)
4674 elif field_type.startswith('reference'):
4675 if field.notnull:
4676 attr = dict(required=True)
4677 referenced = field_type[10:].strip()
4678 ftype = self.types[field_type[:9]](referenced, **attr)
4679 elif field_type.startswith('list:reference'):
4680 if field.notnull:
4681 attr['required'] = True
4682 referenced = field_type[15:].strip()
4683 ftype = self.types[field_type[:14]](**attr)
4684 elif field_type.startswith('list:'):
4685 ftype = self.types[field_type](**attr)
4686 elif not field_type in self.types\
4687 or not self.types[field_type]:
4688 raise SyntaxError('Field: unknown field type: %s' % field_type)
4689 else:
4690 ftype = self.types[field_type](**attr)
4691 myfields[field.name] = ftype
4692 if not polymodel:
4693 table._tableobj = classobj(table._tablename, (gae.Model, ), myfields)
4694 elif polymodel==True:
4695 table._tableobj = classobj(table._tablename, (PolyModel, ), myfields)
4696 elif isinstance(polymodel,Table):
4697 table._tableobj = classobj(table._tablename, (polymodel._tableobj, ), myfields)
4698 else:
4699 raise SyntaxError("polymodel must be None, True, a table or a tablename")
4700 return None
4701
4702 - def expand(self,expression,field_type=None):
4703 if isinstance(expression,Field):
4704 if expression.type in ('text', 'blob', 'json'):
4705 raise SyntaxError('AppEngine does not index by: %s' % expression.type)
4706 return expression.name
4707 elif isinstance(expression, (Expression, Query)):
4708 if not expression.second is None:
4709 return expression.op(expression.first, expression.second)
4710 elif not expression.first is None:
4711 return expression.op(expression.first)
4712 else:
4713 return expression.op()
4714 elif field_type:
4715 return self.represent(expression,field_type)
4716 elif isinstance(expression,(list,tuple)):
4717 return ','.join([self.represent(item,field_type) for item in expression])
4718 else:
4719 return str(expression)
4720
4721
4722 - def AND(self,first,second):
4728
4729 - def EQ(self,first,second=None):
4730 if isinstance(second, Key):
4731 return [GAEF(first.name,'=',second,lambda a,b:a==b)]
4732 return [GAEF(first.name,'=',self.represent(second,first.type),lambda a,b:a==b)]
4733
4734 - def NE(self,first,second=None):
4735 if first.type != 'id':
4736 return [GAEF(first.name,'!=',self.represent(second,first.type),lambda a,b:a!=b)]
4737 else:
4738 if not second is None:
4739 second = Key.from_path(first._tablename, long(second))
4740 return [GAEF(first.name,'!=',second,lambda a,b:a!=b)]
4741
4742 - def LT(self,first,second=None):
4743 if first.type != 'id':
4744 return [GAEF(first.name,'<',self.represent(second,first.type),lambda a,b:a<b)]
4745 else:
4746 second = Key.from_path(first._tablename, long(second))
4747 return [GAEF(first.name,'<',second,lambda a,b:a<b)]
4748
4749 - def LE(self,first,second=None):
4750 if first.type != 'id':
4751 return [GAEF(first.name,'<=',self.represent(second,first.type),lambda a,b:a<=b)]
4752 else:
4753 second = Key.from_path(first._tablename, long(second))
4754 return [GAEF(first.name,'<=',second,lambda a,b:a<=b)]
4755
4756 - def GT(self,first,second=None):
4757 if first.type != 'id' or second==0 or second == '0':
4758 return [GAEF(first.name,'>',self.represent(second,first.type),lambda a,b:a>b)]
4759 else:
4760 second = Key.from_path(first._tablename, long(second))
4761 return [GAEF(first.name,'>',second,lambda a,b:a>b)]
4762
4763 - def GE(self,first,second=None):
4764 if first.type != 'id':
4765 return [GAEF(first.name,'>=',self.represent(second,first.type),lambda a,b:a>=b)]
4766 else:
4767 second = Key.from_path(first._tablename, long(second))
4768 return [GAEF(first.name,'>=',second,lambda a,b:a>=b)]
4769
4772
4773 - def COMMA(self,first,second):
4775
4776 - def BELONGS(self,first,second=None):
4777 if not isinstance(second,(list, tuple)):
4778 raise SyntaxError("Not supported")
4779 if first.type != 'id':
4780 return [GAEF(first.name,'in',self.represent(second,first.type),lambda a,b:a in b)]
4781 else:
4782 second = [Key.from_path(first._tablename, int(i)) for i in second]
4783 return [GAEF(first.name,'in',second,lambda a,b:a in b)]
4784
4785 - def CONTAINS(self,first,second,case_sensitive=False):
4790
4791 - def NOT(self,first):
4792 nops = { self.EQ: self.NE,
4793 self.NE: self.EQ,
4794 self.LT: self.GE,
4795 self.GT: self.LE,
4796 self.LE: self.GT,
4797 self.GE: self.LT}
4798 if not isinstance(first,Query):
4799 raise SyntaxError("Not suported")
4800 nop = nops.get(first.op,None)
4801 if not nop:
4802 raise SyntaxError("Not suported %s" % first.op.__name__)
4803 first.op = nop
4804 return self.expand(first)
4805
4808
4809 - def select_raw(self,query,fields=None,attributes=None):
4810 db = self.db
4811 fields = fields or []
4812 attributes = attributes or {}
4813 args_get = attributes.get
4814 new_fields = []
4815 for item in fields:
4816 if isinstance(item,SQLALL):
4817 new_fields += item._table
4818 else:
4819 new_fields.append(item)
4820 fields = new_fields
4821 if query:
4822 tablename = self.get_table(query)
4823 elif fields:
4824 tablename = fields[0].tablename
4825 query = db._adapter.id_query(fields[0].table)
4826 else:
4827 raise SyntaxError("Unable to determine a tablename")
4828
4829 if query:
4830 if use_common_filters(query):
4831 query = self.common_filter(query,[tablename])
4832
4833
4834 tableobj = db[tablename]._tableobj
4835 filters = self.expand(query)
4836
4837 projection = None
4838 if len(db[tablename].fields) == len(fields):
4839
4840 projection = None
4841 elif args_get('projection') == True:
4842 projection = []
4843 for f in fields:
4844 if f.type in ['text', 'blob', 'json']:
4845 raise SyntaxError(
4846 "text and blob field types not allowed in projection queries")
4847 else:
4848 projection.append(f.name)
4849 elif args_get('filterfields') == True:
4850 projection = []
4851 for f in fields:
4852 projection.append(f.name)
4853
4854
4855
4856 query_projection = [
4857 p for p in projection if \
4858 p != db[tablename]._id.name] if projection and \
4859 args_get('projection') == True\
4860 else None
4861
4862 cursor = None
4863 if isinstance(args_get('reusecursor'), str):
4864 cursor = args_get('reusecursor')
4865 items = gae.Query(tableobj, projection=query_projection,
4866 cursor=cursor)
4867
4868 for filter in filters:
4869 if args_get('projection') == True and \
4870 filter.name in query_projection and \
4871 filter.op in ['=', '<=', '>=']:
4872 raise SyntaxError(
4873 "projection fields cannot have equality filters")
4874 if filter.name=='__key__' and filter.op=='>' and filter.value==0:
4875 continue
4876 elif filter.name=='__key__' and filter.op=='=':
4877 if filter.value==0:
4878 items = []
4879 elif isinstance(filter.value, Key):
4880
4881
4882
4883 item = tableobj.get(filter.value)
4884 items = (item and [item]) or []
4885 else:
4886
4887
4888
4889 item = tableobj.get_by_id(filter.value)
4890 items = (item and [item]) or []
4891 elif isinstance(items,list):
4892 items = [i for i in items if filter.apply(
4893 getattr(item,filter.name),filter.value)]
4894 else:
4895 if filter.name=='__key__' and filter.op != 'in':
4896 items.order('__key__')
4897 items = items.filter('%s %s' % (filter.name,filter.op),
4898 filter.value)
4899 if not isinstance(items,list):
4900 if args_get('left', None):
4901 raise SyntaxError('Set: no left join in appengine')
4902 if args_get('groupby', None):
4903 raise SyntaxError('Set: no groupby in appengine')
4904 orderby = args_get('orderby', False)
4905 if orderby:
4906
4907 if isinstance(orderby, (list, tuple)):
4908 orderby = xorify(orderby)
4909 if isinstance(orderby,Expression):
4910 orderby = self.expand(orderby)
4911 orders = orderby.split(', ')
4912 for order in orders:
4913 order={'-id':'-__key__','id':'__key__'}.get(order,order)
4914 items = items.order(order)
4915 if args_get('limitby', None):
4916 (lmin, lmax) = attributes['limitby']
4917 (limit, offset) = (lmax - lmin, lmin)
4918 rows = items.fetch(limit,offset=offset)
4919
4920
4921 if args_get('reusecursor'):
4922 db['_lastcursor'] = items.cursor()
4923 items = rows
4924 return (items, tablename, projection or db[tablename].fields)
4925
4926 - def select(self,query,fields,attributes):
4927 """
4928 This is the GAE version of select. some notes to consider:
4929 - db['_lastsql'] is not set because there is not SQL statement string
4930 for a GAE query
4931 - 'nativeRef' is a magical fieldname used for self references on GAE
4932 - optional attribute 'projection' when set to True will trigger
4933 use of the GAE projection queries. note that there are rules for
4934 what is accepted imposed by GAE: each field must be indexed,
4935 projection queries cannot contain blob or text fields, and you
4936 cannot use == and also select that same field. see https://developers.google.com/appengine/docs/python/datastore/queries#Query_Projection
4937 - optional attribute 'filterfields' when set to True web2py will only
4938 parse the explicitly listed fields into the Rows object, even though
4939 all fields are returned in the query. This can be used to reduce
4940 memory usage in cases where true projection queries are not
4941 usable.
4942 - optional attribute 'reusecursor' allows use of cursor with queries
4943 that have the limitby attribute. Set the attribute to True for the
4944 first query, set it to the value of db['_lastcursor'] to continue
4945 a previous query. The user must save the cursor value between
4946 requests, and the filters must be identical. It is up to the user
4947 to follow google's limitations: https://developers.google.com/appengine/docs/python/datastore/queries#Query_Cursors
4948 """
4949
4950 (items, tablename, fields) = self.select_raw(query,fields,attributes)
4951
4952 rows = [[(t==self.db[tablename]._id.name and item) or \
4953 (t=='nativeRef' and item) or getattr(item, t) \
4954 for t in fields] for item in items]
4955 colnames = ['%s.%s' % (tablename, t) for t in fields]
4956 processor = attributes.get('processor',self.parse)
4957 return processor(rows,fields,colnames,False)
4958
4959 - def count(self,query,distinct=None,limit=None):
4960 if distinct:
4961 raise RuntimeError("COUNT DISTINCT not supported")
4962 (items, tablename, fields) = self.select_raw(query)
4963
4964 try:
4965 return len(items)
4966 except TypeError:
4967 return items.count(limit=limit)
4968
4969 - def delete(self,tablename, query):
4970 """
4971 This function was changed on 2010-05-04 because according to
4972 http://code.google.com/p/googleappengine/issues/detail?id=3119
4973 GAE no longer supports deleting more than 1000 records.
4974 """
4975
4976 (items, tablename, fields) = self.select_raw(query)
4977
4978 if not isinstance(items,list):
4979
4980
4981 leftitems = items.fetch(1000, keys_only=True)
4982 counter = 0
4983 while len(leftitems):
4984 counter += len(leftitems)
4985 gae.delete(leftitems)
4986 leftitems = items.fetch(1000, keys_only=True)
4987 else:
4988 counter = len(items)
4989 gae.delete(items)
4990 return counter
4991
4992 - def update(self,tablename,query,update_fields):
4993
4994 (items, tablename, fields) = self.select_raw(query)
4995 counter = 0
4996 for item in items:
4997 for field, value in update_fields:
4998 setattr(item, field.name, self.represent(value,field.type))
4999 item.put()
5000 counter += 1
5001 LOGGER.info(str(counter))
5002 return counter
5003
5004 - def insert(self,table,fields):
5005 dfields=dict((f.name,self.represent(v,f.type)) for f,v in fields)
5006
5007 tmp = table._tableobj(**dfields)
5008 tmp.put()
5009 rid = Reference(tmp.key().id())
5010 (rid._table, rid._record, rid._gaekey) = (table, None, tmp.key())
5011 return rid
5012
5014 parsed_items = []
5015 for item in items:
5016 dfields=dict((f.name,self.represent(v,f.type)) for f,v in item)
5017 parsed_items.append(table._tableobj(**dfields))
5018 gae.put(parsed_items)
5019 return True
5020
5022 return uuid.UUID(uuidv).int
5023
5025 return str(uuid.UUID(int=n))
5026
5028 drivers = ('couchdb',)
5029
5030 uploads_in_blob = True
5031 types = {
5032 'boolean': bool,
5033 'string': str,
5034 'text': str,
5035 'json': str,
5036 'password': str,
5037 'blob': str,
5038 'upload': str,
5039 'integer': long,
5040 'bigint': long,
5041 'float': float,
5042 'double': float,
5043 'date': datetime.date,
5044 'time': datetime.time,
5045 'datetime': datetime.datetime,
5046 'id': long,
5047 'reference': long,
5048 'list:string': list,
5049 'list:integer': list,
5050 'list:reference': list,
5051 }
5052
5054 - def file_open(self, filename, mode='rb', lock=True): pass
5056
5057 - def expand(self,expression,field_type=None):
5058 if isinstance(expression,Field):
5059 if expression.type=='id':
5060 return "%s._id" % expression.tablename
5061 return BaseAdapter.expand(self,expression,field_type)
5062
5063 - def AND(self,first,second):
5065
5066 - def OR(self,first,second):
5068
5069 - def EQ(self,first,second):
5073
5074 - def NE(self,first,second):
5078
5079 - def COMMA(self,first,second):
5081
5083 value = NoSQLAdapter.represent(self, obj, fieldtype)
5084 if fieldtype=='id':
5085 return repr(str(long(value)))
5086 elif fieldtype in ('date','time','datetime','boolean'):
5087 return serializers.json(value)
5088 return repr(not isinstance(value,unicode) and value \
5089 or value and value.encode('utf8'))
5090
5091 - def __init__(self,db,uri='couchdb://127.0.0.1:5984',
5092 pool_size=0,folder=None,db_codec ='UTF-8',
5093 credential_decoder=IDENTITY, driver_args={},
5094 adapter_args={}, do_connect=True, after_connection=None):
5095 self.db = db
5096 self.uri = uri
5097 if do_connect: self.find_driver(adapter_args)
5098 self.dbengine = 'couchdb'
5099 self.folder = folder
5100 db['_lastsql'] = ''
5101 self.db_codec = 'UTF-8'
5102 self._after_connection = after_connection
5103 self.pool_size = pool_size
5104
5105 url='http://'+uri[10:]
5106 def connector(url=url,driver_args=driver_args):
5107 return self.driver.Server(url,**driver_args)
5108 self.reconnect(connector,cursor=False)
5109
5110 - def create_table(self, table, migrate=True, fake_migrate=False, polymodel=None):
5111 if migrate:
5112 try:
5113 self.connection.create(table._tablename)
5114 except:
5115 pass
5116
5117 - def insert(self,table,fields):
5124
5125 - def _select(self,query,fields,attributes):
5126 if not isinstance(query,Query):
5127 raise SyntaxError("Not Supported")
5128 for key in set(attributes.keys())-SELECT_ARGS:
5129 raise SyntaxError('invalid select attribute: %s' % key)
5130 new_fields=[]
5131 for item in fields:
5132 if isinstance(item,SQLALL):
5133 new_fields += item._table
5134 else:
5135 new_fields.append(item)
5136 def uid(fd):
5137 return fd=='id' and '_id' or fd
5138 def get(row,fd):
5139 return fd=='id' and long(row['_id']) or row.get(fd,None)
5140 fields = new_fields
5141 tablename = self.get_table(query)
5142 fieldnames = [f.name for f in (fields or self.db[tablename])]
5143 colnames = ['%s.%s' % (tablename,k) for k in fieldnames]
5144 fields = ','.join(['%s.%s' % (tablename,uid(f)) for f in fieldnames])
5145 fn="(function(%(t)s){if(%(query)s)emit(%(order)s,[%(fields)s]);})" %\
5146 dict(t=tablename,
5147 query=self.expand(query),
5148 order='%s._id' % tablename,
5149 fields=fields)
5150 return fn, colnames
5151
5152 - def select(self,query,fields,attributes):
5153 if not isinstance(query,Query):
5154 raise SyntaxError("Not Supported")
5155 fn, colnames = self._select(query,fields,attributes)
5156 tablename = colnames[0].split('.')[0]
5157 ctable = self.connection[tablename]
5158 rows = [cols['value'] for cols in ctable.query(fn)]
5159 processor = attributes.get('processor',self.parse)
5160 return processor(rows,fields,colnames,False)
5161
5162 - def delete(self,tablename,query):
5163 if not isinstance(query,Query):
5164 raise SyntaxError("Not Supported")
5165 if query.first.type=='id' and query.op==self.EQ:
5166 id = query.second
5167 tablename = query.first.tablename
5168 assert(tablename == query.first.tablename)
5169 ctable = self.connection[tablename]
5170 try:
5171 del ctable[str(id)]
5172 return 1
5173 except couchdb.http.ResourceNotFound:
5174 return 0
5175 else:
5176 tablename = self.get_table(query)
5177 rows = self.select(query,[self.db[tablename]._id],{})
5178 ctable = self.connection[tablename]
5179 for row in rows:
5180 del ctable[str(row.id)]
5181 return len(rows)
5182
5183 - def update(self,tablename,query,fields):
5184 if not isinstance(query,Query):
5185 raise SyntaxError("Not Supported")
5186 if query.first.type=='id' and query.op==self.EQ:
5187 id = query.second
5188 tablename = query.first.tablename
5189 ctable = self.connection[tablename]
5190 try:
5191 doc = ctable[str(id)]
5192 for key,value in fields:
5193 doc[key.name] = self.represent(value,self.db[tablename][key.name].type)
5194 ctable.save(doc)
5195 return 1
5196 except couchdb.http.ResourceNotFound:
5197 return 0
5198 else:
5199 tablename = self.get_table(query)
5200 rows = self.select(query,[self.db[tablename]._id],{})
5201 ctable = self.connection[tablename]
5202 table = self.db[tablename]
5203 for row in rows:
5204 doc = ctable[str(row.id)]
5205 for key,value in fields:
5206 doc[key.name] = self.represent(value,table[key.name].type)
5207 ctable.save(doc)
5208 return len(rows)
5209
5210 - def count(self,query,distinct=None):
5211 if distinct:
5212 raise RuntimeError("COUNT DISTINCT not supported")
5213 if not isinstance(query,Query):
5214 raise SyntaxError("Not Supported")
5215 tablename = self.get_table(query)
5216 rows = self.select(query,[self.db[tablename]._id],{})
5217 return len(rows)
5218
5220 """
5221 validates that the given text is clean: only contains [0-9a-zA-Z_]
5222 """
5223 if not REGEX_ALPHANUMERIC.match(text):
5224 raise SyntaxError('invalid table or field name: %s' % text)
5225 return text
5226
5228 native_json = True
5229 drivers = ('pymongo',)
5230
5231 uploads_in_blob = True
5232
5233 types = {
5234 'boolean': bool,
5235 'string': str,
5236 'text': str,
5237 'json': str,
5238 'password': str,
5239 'blob': str,
5240 'upload': str,
5241 'integer': long,
5242 'bigint': long,
5243 'float': float,
5244 'double': float,
5245 'date': datetime.date,
5246 'time': datetime.time,
5247 'datetime': datetime.datetime,
5248 'id': long,
5249 'reference': long,
5250 'list:string': list,
5251 'list:integer': list,
5252 'list:reference': list,
5253 }
5254
5255 error_messages = {"javascript_needed": "This must yet be replaced" +
5256 " with javascript in order to work."}
5257
5258 - def __init__(self,db,uri='mongodb://127.0.0.1:5984/db',
5259 pool_size=0, folder=None, db_codec ='UTF-8',
5260 credential_decoder=IDENTITY, driver_args={},
5261 adapter_args={}, do_connect=True, after_connection=None):
5262
5263 self.db = db
5264 self.uri = uri
5265 if do_connect: self.find_driver(adapter_args)
5266 import random
5267 from bson.objectid import ObjectId
5268 from bson.son import SON
5269 import pymongo.uri_parser
5270
5271 m = pymongo.uri_parser.parse_uri(uri)
5272
5273 self.SON = SON
5274 self.ObjectId = ObjectId
5275 self.random = random
5276
5277 self.dbengine = 'mongodb'
5278 self.folder = folder
5279 db['_lastsql'] = ''
5280 self.db_codec = 'UTF-8'
5281 self._after_connection = after_connection
5282 self.pool_size = pool_size
5283
5284
5285 self.minimumreplication = adapter_args.get('minimumreplication',0)
5286
5287
5288
5289
5290 self.safe = adapter_args.get('safe',True)
5291
5292 if isinstance(m,tuple):
5293 m = {"database" : m[1]}
5294 if m.get('database')==None:
5295 raise SyntaxError("Database is required!")
5296
5297 def connector(uri=self.uri,m=m):
5298
5299 if hasattr(self.driver, "MongoClient"):
5300 Connection = self.driver.MongoClient
5301 else:
5302 Connection = self.driver.Connection
5303 return Connection(uri)[m.get('database')]
5304
5305 self.reconnect(connector,cursor=False)
5306
5308 """ Convert input to a valid Mongodb ObjectId instance
5309
5310 self.object_id("<random>") -> ObjectId (not unique) instance """
5311 if not arg:
5312 arg = 0
5313 if isinstance(arg, basestring):
5314
5315 rawhex = len(arg.replace("0x", "").replace("L", "")) == 24
5316 if arg.isdigit() and (not rawhex):
5317 arg = int(arg)
5318 elif arg == "<random>":
5319 arg = int("0x%sL" % \
5320 "".join([self.random.choice("0123456789abcdef") \
5321 for x in range(24)]), 0)
5322 elif arg.isalnum():
5323 if not arg.startswith("0x"):
5324 arg = "0x%s" % arg
5325 try:
5326 arg = int(arg, 0)
5327 except ValueError, e:
5328 raise ValueError(
5329 "invalid objectid argument string: %s" % e)
5330 else:
5331 raise ValueError("Invalid objectid argument string. " +
5332 "Requires an integer or base 16 value")
5333 elif isinstance(arg, self.ObjectId):
5334 return arg
5335
5336 if not isinstance(arg, (int, long)):
5337 raise TypeError("object_id argument must be of type " +
5338 "ObjectId or an objectid representable integer")
5339 if arg == 0:
5340 hexvalue = "".zfill(24)
5341 else:
5342 hexvalue = hex(arg)[2:].replace("L", "")
5343 return self.ObjectId(hexvalue)
5344
5346
5347 if isinstance(value, self.ObjectId):
5348 value = long(str(value), 16)
5349 return super(MongoDBAdapter,
5350 self).parse_reference(value, field_type)
5351
5352 - def parse_id(self, value, field_type):
5353 if isinstance(value, self.ObjectId):
5354 value = long(str(value), 16)
5355 return super(MongoDBAdapter,
5356 self).parse_id(value, field_type)
5357
5359
5360 if isinstance(obj, self.ObjectId):
5361 value = obj
5362 else:
5363 value = NoSQLAdapter.represent(self, obj, fieldtype)
5364
5365 if fieldtype =='date':
5366 if value == None:
5367 return value
5368
5369 t = datetime.time(0, 0, 0)
5370
5371
5372 return datetime.datetime.combine(value, t)
5373 elif fieldtype == 'time':
5374 if value == None:
5375 return value
5376
5377 d = datetime.date(2000, 1, 1)
5378
5379
5380 return datetime.datetime.combine(d, value)
5381 elif fieldtype == "blob":
5382 from bson import Binary
5383 if not isinstance(value, Binary):
5384 return Binary(value)
5385 return value
5386 elif (isinstance(fieldtype, basestring) and
5387 fieldtype.startswith('list:')):
5388 if fieldtype.startswith('list:reference'):
5389 newval = []
5390 for v in value:
5391 newval.append(self.object_id(v))
5392 return newval
5393 return value
5394 elif ((isinstance(fieldtype, basestring) and
5395 fieldtype.startswith("reference")) or
5396 (isinstance(fieldtype, Table)) or fieldtype=="id"):
5397 value = self.object_id(value)
5398 return value
5399
5400 - def create_table(self, table, migrate=True, fake_migrate=False,
5401 polymodel=None, isCapped=False):
5402 if isCapped:
5403 raise RuntimeError("Not implemented")
5404
5405 - def count(self, query, distinct=None, snapshot=True):
5406 if distinct:
5407 raise RuntimeError("COUNT DISTINCT not supported")
5408 if not isinstance(query,Query):
5409 raise SyntaxError("Not Supported")
5410 tablename = self.get_table(query)
5411 return long(self.select(query,[self.db[tablename]._id], {},
5412 count=True,snapshot=snapshot)['count'])
5413
5414
5415
5416
5417
5418 - def expand(self, expression, field_type=None):
5419 if isinstance(expression, Query):
5420
5421
5422
5423
5424
5425 if isinstance(expression.first,Field) and \
5426 ((expression.first.type == 'id') or \
5427 ("reference" in expression.first.type)):
5428 if expression.first.type == 'id':
5429 expression.first.name = '_id'
5430
5431 if isinstance(expression.second, (tuple, list, set)):
5432 expression.second = [self.object_id(item) for
5433 item in expression.second]
5434 else:
5435 expression.second = self.object_id(expression.second)
5436 result = expression.op(expression.first, expression.second)
5437
5438 if isinstance(expression, Field):
5439 if expression.type=='id':
5440 result = "_id"
5441 else:
5442 result = expression.name
5443 elif isinstance(expression, (Expression, Query)):
5444 if not expression.second is None:
5445 result = expression.op(expression.first, expression.second)
5446 elif not expression.first is None:
5447 result = expression.op(expression.first)
5448 elif not isinstance(expression.op, str):
5449 result = expression.op()
5450 else:
5451 result = expression.op
5452 elif field_type:
5453 result = self.represent(expression,field_type)
5454 elif isinstance(expression,(list,tuple)):
5455 result = ','.join(self.represent(item,field_type) for
5456 item in expression)
5457 else:
5458 result = expression
5459 return result
5460
5461 - def drop(self, table, mode=''):
5464
5465 - def truncate(self, table, mode, safe=None):
5466 if safe == None:
5467 safe=self.safe
5468 ctable = self.connection[table._tablename]
5469 ctable.remove(None, safe=True)
5470
5471 - def _select(self, query, fields, attributes):
5472 if 'for_update' in attributes:
5473 logging.warn('mongodb does not support for_update')
5474 for key in set(attributes.keys())-set(('limitby',
5475 'orderby','for_update')):
5476 if attributes[key]!=None:
5477 logging.warn('select attribute not implemented: %s' % key)
5478
5479 new_fields=[]
5480 mongosort_list = []
5481
5482
5483 orderby = attributes.get('orderby', False)
5484 limitby = attributes.get('limitby', False)
5485
5486 if orderby:
5487 if isinstance(orderby, (list, tuple)):
5488 orderby = xorify(orderby)
5489
5490
5491 for f in self.expand(orderby).split(','):
5492 if f.startswith('-'):
5493 mongosort_list.append((f[1:], -1))
5494 else:
5495 mongosort_list.append((f, 1))
5496 if limitby:
5497 limitby_skip, limitby_limit = limitby[0], int(limitby[1])
5498 else:
5499 limitby_skip = limitby_limit = 0
5500
5501 mongofields_dict = self.SON()
5502 mongoqry_dict = {}
5503 for item in fields:
5504 if isinstance(item, SQLALL):
5505 new_fields += item._table
5506 else:
5507 new_fields.append(item)
5508 fields = new_fields
5509 if isinstance(query,Query):
5510 tablename = self.get_table(query)
5511 elif len(fields) != 0:
5512 tablename = fields[0].tablename
5513 else:
5514 raise SyntaxError("The table name could not be found in " +
5515 "the query nor from the select statement.")
5516 mongoqry_dict = self.expand(query)
5517 fields = fields or self.db[tablename]
5518 for field in fields:
5519 mongofields_dict[field.name] = 1
5520
5521 return tablename, mongoqry_dict, mongofields_dict, mongosort_list, \
5522 limitby_limit, limitby_skip
5523
5524 - def select(self, query, fields, attributes, count=False,
5525 snapshot=False):
5526
5527 tablename, mongoqry_dict, mongofields_dict, mongosort_list, \
5528 limitby_limit, limitby_skip = self._select(query, fields, attributes)
5529 ctable = self.connection[tablename]
5530
5531 if count:
5532 return {'count' : ctable.find(
5533 mongoqry_dict, mongofields_dict,
5534 skip=limitby_skip, limit=limitby_limit,
5535 sort=mongosort_list, snapshot=snapshot).count()}
5536 else:
5537
5538 mongo_list_dicts = ctable.find(mongoqry_dict,
5539 mongofields_dict, skip=limitby_skip,
5540 limit=limitby_limit, sort=mongosort_list,
5541 snapshot=snapshot)
5542 rows = []
5543
5544
5545 colnames = []
5546 newnames = []
5547 for field in fields:
5548 colname = str(field)
5549 colnames.append(colname)
5550 tablename, fieldname = colname.split(".")
5551 if fieldname == "_id":
5552
5553 field.name = "id"
5554 newnames.append(".".join((tablename, field.name)))
5555
5556 for record in mongo_list_dicts:
5557 row=[]
5558 for colname in colnames:
5559 tablename, fieldname = colname.split(".")
5560
5561
5562 if fieldname == "id": fieldname = "_id"
5563 if fieldname in record:
5564 value = record[fieldname]
5565 else:
5566 value = None
5567 row.append(value)
5568 rows.append(row)
5569
5570 processor = attributes.get('processor', self.parse)
5571 result = processor(rows, fields, newnames, False)
5572 return result
5573
5574 - def _insert(self, table, fields):
5582
5583
5584
5585
5586 - def insert(self, table, fields, safe=None):
5593
5594
5595 - def _update(self, tablename, query, fields):
5596 if not isinstance(query, Query):
5597 raise SyntaxError("Not Supported")
5598 filter = None
5599 if query:
5600 filter = self.expand(query)
5601
5602 modify = {'$set': dict((k.name, self.represent(v, k.type)) for
5603 k, v in fields if (not k.name in ("_id", "id")))}
5604 return modify, filter
5605
5606 - def update(self, tablename, query, fields, safe=None):
5607 if safe == None:
5608 safe = self.safe
5609
5610
5611 if not isinstance(query, Query):
5612 raise RuntimeError("Not implemented")
5613 amount = self.count(query, False)
5614 modify, filter = self._update(tablename, query, fields)
5615 try:
5616 result = self.connection[tablename].update(filter,
5617 modify, multi=True, safe=safe)
5618 if safe:
5619 try:
5620
5621 return result["n"]
5622 except (KeyError, AttributeError, TypeError):
5623 return amount
5624 else:
5625 return amount
5626 except Exception, e:
5627
5628 raise RuntimeError("uncaught exception when updating rows: %s" % e)
5629
5630 - def _delete(self, tablename, query):
5631 if not isinstance(query, Query):
5632 raise RuntimeError("query type %s is not supported" % \
5633 type(query))
5634 return self.expand(query)
5635
5636 - def delete(self, tablename, query, safe=None):
5637 if safe is None:
5638 safe = self.safe
5639 amount = 0
5640 amount = self.count(query, False)
5641 filter = self._delete(tablename, query)
5642 self.connection[tablename].remove(filter, safe=safe)
5643 return amount
5644
5646 return [self.insert(table,item) for item in items]
5647
5648
5652
5653
5654 - def NOT(self, first):
5655 result = {}
5656 result["$not"] = self.expand(first)
5657 return result
5658
5659 - def AND(self,first,second):
5664
5665 - def OR(self,first,second):
5666
5667 result = {}
5668 f = self.expand(first)
5669 s = self.expand(second)
5670 result['$or'] = [f,s]
5671 return result
5672
5673 - def BELONGS(self, first, second):
5674 if isinstance(second, str):
5675 return {self.expand(first) : {"$in" : [ second[:-1]]} }
5676 elif second==[] or second==() or second==set():
5677 return {1:0}
5678 items = [self.expand(item, first.type) for item in second]
5679 return {self.expand(first) : {"$in" : items} }
5680
5681 - def EQ(self,first,second=None):
5682 result = {}
5683 result[self.expand(first)] = self.expand(second)
5684 return result
5685
5686 - def NE(self, first, second=None):
5687 result = {}
5688 result[self.expand(first)] = {'$ne': self.expand(second)}
5689 return result
5690
5691 - def LT(self,first,second=None):
5692 if second is None:
5693 raise RuntimeError("Cannot compare %s < None" % first)
5694 result = {}
5695 result[self.expand(first)] = {'$lt': self.expand(second)}
5696 return result
5697
5698 - def LE(self,first,second=None):
5699 if second is None:
5700 raise RuntimeError("Cannot compare %s <= None" % first)
5701 result = {}
5702 result[self.expand(first)] = {'$lte': self.expand(second)}
5703 return result
5704
5705 - def GT(self,first,second):
5706 result = {}
5707 result[self.expand(first)] = {'$gt': self.expand(second)}
5708 return result
5709
5710 - def GE(self,first,second=None):
5711 if second is None:
5712 raise RuntimeError("Cannot compare %s >= None" % first)
5713 result = {}
5714 result[self.expand(first)] = {'$gte': self.expand(second)}
5715 return result
5716
5717 - def ADD(self, first, second):
5721
5722 - def SUB(self, first, second):
5726
5727 - def MUL(self, first, second):
5731
5732 - def DIV(self, first, second):
5736
5737 - def MOD(self, first, second):
5741
5742 - def AS(self, first, second):
5743 raise NotImplementedError(self.error_messages["javascript_needed"])
5744 return '%s AS %s' % (self.expand(first), second)
5745
5746
5747
5748
5749 - def ON(self, first, second):
5750 raise NotImplementedError("This is not possible in NoSQL" +
5751 " but can be simulated with a wrapper.")
5752 return '%s ON %s' % (self.expand(first), self.expand(second))
5753
5754
5755
5756
5757 - def COMMA(self, first, second):
5759
5760 - def LIKE(self, first, second):
5764
5766
5767 return {self.expand(first): ('/^%s/' % \
5768 self.expand(second, 'string'))}
5769
5771
5772 return {self.expand(first): ('/%s^/' % \
5773 self.expand(second, 'string'))}
5774
5775 - def CONTAINS(self, first, second, case_sensitive=False):
5776
5777
5778
5779 val = second if isinstance(second,self.ObjectId) else \
5780 {'$regex':".*" + re.escape(self.expand(second, 'string')) + ".*"}
5781 return {self.expand(first) : val}
5782
5783 - def LIKE(self, first, second):
5788
5789
5791
5792 import re
5793 return {self.expand(first): {'$regex' : '^' +
5794 re.escape(self.expand(second,
5795 'string'))}}
5796
5797
5799
5800
5801
5802
5803 import re
5804 return {self.expand(first): {'$regex': \
5805 re.escape(self.expand(second, 'string')) + '$'}}
5806
5807
5808 - def CONTAINS(self, first, second, case_sensitive=False):
5809
5810
5811
5812
5813 return {self.expand(first) : {'$regex': \
5814 ".*" + re.escape(self.expand(second, 'string')) + ".*"}}
5815
5818 drivers = ('imaplib',)
5819
5820 """ IMAP server adapter
5821
5822 This class is intended as an interface with
5823 email IMAP servers to perform simple queries in the
5824 web2py DAL query syntax, so email read, search and
5825 other related IMAP mail services (as those implemented
5826 by brands like Google(r), and Yahoo!(r)
5827 can be managed from web2py applications.
5828
5829 The code uses examples by Yuji Tomita on this post:
5830 http://yuji.wordpress.com/2011/06/22/python-imaplib-imap-example-with-gmail/#comment-1137
5831 and is based in docs for Python imaplib, python email
5832 and email IETF's (i.e. RFC2060 and RFC3501)
5833
5834 This adapter was tested with a small set of operations with Gmail(r). Other
5835 services requests could raise command syntax and response data issues.
5836
5837 It creates its table and field names "statically",
5838 meaning that the developer should leave the table and field
5839 definitions to the DAL instance by calling the adapter's
5840 .define_tables() method. The tables are defined with the
5841 IMAP server mailbox list information.
5842
5843 .define_tables() returns a dictionary mapping dal tablenames
5844 to the server mailbox names with the following structure:
5845
5846 {<tablename>: str <server mailbox name>}
5847
5848 Here is a list of supported fields:
5849
5850 Field Type Description
5851 ################################################################
5852 uid string
5853 answered boolean Flag
5854 created date
5855 content list:string A list of dict text or html parts
5856 to string
5857 cc string
5858 bcc string
5859 size integer the amount of octets of the message*
5860 deleted boolean Flag
5861 draft boolean Flag
5862 flagged boolean Flag
5863 sender string
5864 recent boolean Flag
5865 seen boolean Flag
5866 subject string
5867 mime string The mime header declaration
5868 email string The complete RFC822 message**
5869 attachments <type list> Each non text part as dict
5870 encoding string The main detected encoding
5871
5872 *At the application side it is measured as the length of the RFC822
5873 message string
5874
5875 WARNING: As row id's are mapped to email sequence numbers,
5876 make sure your imap client web2py app does not delete messages
5877 during select or update actions, to prevent
5878 updating or deleting different messages.
5879 Sequence numbers change whenever the mailbox is updated.
5880 To avoid this sequence numbers issues, it is recommended the use
5881 of uid fields in query references (although the update and delete
5882 in separate actions rule still applies).
5883
5884 # This is the code recommended to start imap support
5885 # at the app's model:
5886
5887 imapdb = DAL("imap://user:password@server:port", pool_size=1) # port 993 for ssl
5888 imapdb.define_tables()
5889
5890 Here is an (incomplete) list of possible imap commands:
5891
5892 # Count today's unseen messages
5893 # smaller than 6000 octets from the
5894 # inbox mailbox
5895
5896 q = imapdb.INBOX.seen == False
5897 q &= imapdb.INBOX.created == datetime.date.today()
5898 q &= imapdb.INBOX.size < 6000
5899 unread = imapdb(q).count()
5900
5901 # Fetch last query messages
5902 rows = imapdb(q).select()
5903
5904 # it is also possible to filter query select results with limitby and
5905 # sequences of mailbox fields
5906
5907 set.select(<fields sequence>, limitby=(<int>, <int>))
5908
5909 # Mark last query messages as seen
5910 messages = [row.uid for row in rows]
5911 seen = imapdb(imapdb.INBOX.uid.belongs(messages)).update(seen=True)
5912
5913 # Delete messages in the imap database that have mails from mr. Gumby
5914
5915 deleted = 0
5916 for mailbox in imapdb.tables
5917 deleted += imapdb(imapdb[mailbox].sender.contains("gumby")).delete()
5918
5919 # It is possible also to mark messages for deletion instead of ereasing them
5920 # directly with set.update(deleted=True)
5921
5922
5923 # This object give access
5924 # to the adapter auto mailbox
5925 # mapped names (which native
5926 # mailbox has what table name)
5927
5928 imapdb.mailboxes <dict> # tablename, server native name pairs
5929
5930 # To retrieve a table native mailbox name use:
5931 imapdb.<table>.mailbox
5932
5933 ### New features v2.4.1:
5934
5935 # Declare mailboxes statically with tablename, name pairs
5936 # This avoids the extra server names retrieval
5937
5938 imapdb.define_tables({"inbox": "INBOX"})
5939
5940 # Selects without content/attachments/email columns will only
5941 # fetch header and flags
5942
5943 imapdb(q).select(imapdb.INBOX.sender, imapdb.INBOX.subject)
5944 """
5945
5946 types = {
5947 'string': str,
5948 'text': str,
5949 'date': datetime.date,
5950 'datetime': datetime.datetime,
5951 'id': long,
5952 'boolean': bool,
5953 'integer': int,
5954 'bigint': long,
5955 'blob': str,
5956 'list:string': str,
5957 }
5958
5959 dbengine = 'imap'
5960
5961 REGEX_URI = re.compile('^(?P<user>[^:]+)(\:(?P<password>[^@]*))?@(?P<host>[^\:@]+)(\:(?P<port>[0-9]+))?$')
5962
5963 - def __init__(self,
5964 db,
5965 uri,
5966 pool_size=0,
5967 folder=None,
5968 db_codec ='UTF-8',
5969 credential_decoder=IDENTITY,
5970 driver_args={},
5971 adapter_args={},
5972 do_connect=True,
5973 after_connection=None):
5974
5975
5976
5977
5978 self.db = db
5979 self.uri = uri
5980 if do_connect: self.find_driver(adapter_args)
5981 self.pool_size=pool_size
5982 self.folder = folder
5983 self.db_codec = db_codec
5984 self._after_connection = after_connection
5985 self.credential_decoder = credential_decoder
5986 self.driver_args = driver_args
5987 self.adapter_args = adapter_args
5988 self.mailbox_size = None
5989 self.static_names = None
5990 self.charset = sys.getfilesystemencoding()
5991
5992 self.imap4 = None
5993 uri = uri.split("://")[1]
5994
5995 """ MESSAGE is an identifier for sequence number"""
5996
5997 self.flags = {'deleted': '\\Deleted', 'draft': '\\Draft',
5998 'flagged': '\\Flagged', 'recent': '\\Recent',
5999 'seen': '\\Seen', 'answered': '\\Answered'}
6000 self.search_fields = {
6001 'id': 'MESSAGE', 'created': 'DATE',
6002 'uid': 'UID', 'sender': 'FROM',
6003 'to': 'TO', 'cc': 'CC',
6004 'bcc': 'BCC', 'content': 'TEXT',
6005 'size': 'SIZE', 'deleted': '\\Deleted',
6006 'draft': '\\Draft', 'flagged': '\\Flagged',
6007 'recent': '\\Recent', 'seen': '\\Seen',
6008 'subject': 'SUBJECT', 'answered': '\\Answered',
6009 'mime': None, 'email': None,
6010 'attachments': None
6011 }
6012
6013 db['_lastsql'] = ''
6014
6015 m = self.REGEX_URI.match(uri)
6016 user = m.group('user')
6017 password = m.group('password')
6018 host = m.group('host')
6019 port = int(m.group('port'))
6020 over_ssl = False
6021 if port==993:
6022 over_ssl = True
6023
6024 driver_args.update(host=host,port=port, password=password, user=user)
6025 def connector(driver_args=driver_args):
6026
6027
6028 if over_ssl:
6029 self.imap4 = self.driver.IMAP4_SSL
6030 else:
6031 self.imap4 = self.driver.IMAP4
6032 connection = self.imap4(driver_args["host"], driver_args["port"])
6033 data = connection.login(driver_args["user"], driver_args["password"])
6034
6035
6036 connection.mailbox_names = None
6037
6038
6039 connection.cursor = lambda : True
6040
6041 return connection
6042
6043 self.db.define_tables = self.define_tables
6044 self.connector = connector
6045 if do_connect: self.reconnect()
6046
6091
6093 last_message = None
6094
6095
6096 if not isinstance(self.connection.mailbox_names, dict):
6097 self.get_mailboxes()
6098 try:
6099 result = self.connection.select(self.connection.mailbox_names[tablename])
6100 last_message = int(result[1][0])
6101 except (IndexError, ValueError, TypeError, KeyError):
6102 e = sys.exc_info()[1]
6103 LOGGER.debug("Error retrieving the last mailbox sequence number. %s" % str(e))
6104 return last_message
6105
6107 if not isinstance(self.connection.mailbox_names, dict):
6108 self.get_mailboxes()
6109
6110
6111 last_message = self.get_last_message(tablename)
6112 result, data = self.connection.uid("search", None, "(ALL)")
6113 uid_list = data[0].strip().split()
6114 if len(uid_list) <= 0:
6115 return None
6116 else:
6117 return (uid_list[0], uid_list[-1])
6118
6120 if add is None:
6121 add = datetime.timedelta()
6122 """ Convert a date object to a string
6123 with d-Mon-Y style for IMAP or the inverse
6124 case
6125
6126 add <timedelta> adds to the date object
6127 """
6128 months = [None, "JAN","FEB","MAR","APR","MAY","JUN",
6129 "JUL", "AUG","SEP","OCT","NOV","DEC"]
6130 if isinstance(date, basestring):
6131
6132 try:
6133 dayname, datestring = date.split(",")
6134 date_list = datestring.strip().split()
6135 year = int(date_list[2])
6136 month = months.index(date_list[1].upper())
6137 day = int(date_list[0])
6138 hms = map(int, date_list[3].split(":"))
6139 return datetime.datetime(year, month, day,
6140 hms[0], hms[1], hms[2]) + add
6141 except (ValueError, AttributeError, IndexError), e:
6142 LOGGER.error("Could not parse date text: %s. %s" %
6143 (date, e))
6144 return None
6145 elif isinstance(date, (datetime.datetime, datetime.date)):
6146 return (date + add).strftime("%d-%b-%Y")
6147 else:
6148 return None
6149
6150 @staticmethod
6152 from email.header import decode_header
6153 text, encoding = decode_header(f)[0]
6154 if encoding:
6155 text = text.decode(encoding).encode('utf-8')
6156 return text
6157
6158 - def encode_text(self, text, charset, errors="replace"):
6159 """ convert text for mail to unicode"""
6160 if text is None:
6161 text = ""
6162 else:
6163 if isinstance(text, str):
6164 if charset is None:
6165 text = unicode(text, "utf-8", errors)
6166 else:
6167 text = unicode(text, charset, errors)
6168 else:
6169 raise Exception("Unsupported mail text type %s" % type(text))
6170 return text.encode("utf-8")
6171
6173 charset = message.get_content_charset()
6174 return charset
6175
6177 """ Query the mail database for mailbox names """
6178 if self.static_names:
6179
6180 self.connection.mailbox_names = self.static_names
6181 return self.static_names.keys()
6182
6183 mailboxes_list = self.connection.list()
6184 self.connection.mailbox_names = dict()
6185 mailboxes = list()
6186 x = 0
6187 for item in mailboxes_list[1]:
6188 x = x + 1
6189 item = item.strip()
6190 if not "NOSELECT" in item.upper():
6191 sub_items = item.split("\"")
6192 sub_items = [sub_item for sub_item in sub_items \
6193 if len(sub_item.strip()) > 0]
6194
6195 mailbox = sub_items[-1]
6196
6197
6198 mailbox_name = re.sub('^[_0-9]*', '', re.sub('[^_\w]','',re.sub('[/ ]','_',mailbox)))
6199 mailboxes.append(mailbox_name)
6200 self.connection.mailbox_names[mailbox_name] = mailbox
6201
6202 return mailboxes
6203
6205 nofield = True
6206 tablename = None
6207 attr = query
6208 while nofield:
6209 if hasattr(attr, "first"):
6210 attr = attr.first
6211 if isinstance(attr, Field):
6212 return attr.tablename
6213 elif isinstance(attr, Query):
6214 pass
6215 else:
6216 return None
6217 else:
6218 return None
6219 return tablename
6220
6222 if self.search_fields.get(flag, None) in self.flags.values():
6223 return True
6224 else:
6225 return False
6226
6228 """
6229 Auto create common IMAP fileds
6230
6231 This function creates fields definitions "statically"
6232 meaning that custom fields as in other adapters should
6233 not be supported and definitions handled on a service/mode
6234 basis (local syntax for Gmail(r), Ymail(r)
6235
6236 Returns a dictionary with tablename, server native mailbox name
6237 pairs.
6238 """
6239 if mailbox_names:
6240
6241 self.static_names = mailbox_names
6242 else:
6243 self.static_names = None
6244 if not isinstance(self.connection.mailbox_names, dict):
6245 self.get_mailboxes()
6246
6247 names = self.connection.mailbox_names.keys()
6248
6249 for name in names:
6250 self.db.define_table("%s" % name,
6251 Field("uid", "string", writable=False),
6252 Field("answered", "boolean"),
6253 Field("created", "datetime", writable=False),
6254 Field("content", list, writable=False),
6255 Field("to", "string", writable=False),
6256 Field("cc", "string", writable=False),
6257 Field("bcc", "string", writable=False),
6258 Field("size", "integer", writable=False),
6259 Field("deleted", "boolean"),
6260 Field("draft", "boolean"),
6261 Field("flagged", "boolean"),
6262 Field("sender", "string", writable=False),
6263 Field("recent", "boolean", writable=False),
6264 Field("seen", "boolean"),
6265 Field("subject", "string", writable=False),
6266 Field("mime", "string", writable=False),
6267 Field("email", "string", writable=False, readable=False),
6268 Field("attachments", list, writable=False, readable=False),
6269 Field("encoding", writable=False)
6270 )
6271
6272
6273
6274 self.db[name].mailbox = \
6275 self.connection.mailbox_names[name]
6276
6277
6278 self.db[name].to.represent = self.db[name].cc.represent = \
6279 self.db[name].bcc.represent = self.db[name].sender.represent = \
6280 self.db[name].subject.represent = self.header_represent
6281
6282
6283 self.db.mailboxes = self.connection.mailbox_names
6284 return self.db.mailboxes
6285
6290
6291 - def _select(self, query, fields, attributes):
6295
6296 - def select(self, query, fields, attributes):
6297 """ Search and Fetch records and return web2py rows
6298 """
6299
6300 if use_common_filters(query):
6301 query = self.common_filter(query, [self.get_query_mailbox(query),])
6302
6303 import email
6304
6305
6306 tablename = None
6307 fetch_results = list()
6308
6309 if isinstance(query, Query):
6310 tablename = self.get_table(query)
6311 mailbox = self.connection.mailbox_names.get(tablename, None)
6312 if mailbox is None:
6313 raise ValueError("Mailbox name not found: %s" % mailbox)
6314 else:
6315
6316 result, selected = self.connection.select(mailbox, True)
6317 if result != "OK":
6318 raise Exception("IMAP error: %s" % selected)
6319 self.mailbox_size = int(selected[0])
6320 search_query = "(%s)" % str(query).strip()
6321 search_result = self.connection.uid("search", None, search_query)
6322
6323 if search_result[0] == "OK":
6324
6325
6326
6327
6328 limitby = attributes.get('limitby', None)
6329 messages_set = search_result[1][0].split()
6330
6331 messages_set.reverse()
6332 if limitby is not None:
6333
6334 messages_set = messages_set[int(limitby[0]):int(limitby[1])]
6335
6336
6337 if any([(field.name in ["content", "size",
6338 "attachments", "email"]) for
6339 field in fields]):
6340 imap_fields = "(RFC822 FLAGS)"
6341 else:
6342 imap_fields = "(RFC822.HEADER FLAGS)"
6343
6344 if len(messages_set) > 0:
6345
6346
6347
6348
6349 for uid in messages_set:
6350
6351 typ, data = self.connection.uid("fetch", uid, imap_fields)
6352 if typ == "OK":
6353 fr = {"message": int(data[0][0].split()[0]),
6354 "uid": long(uid),
6355 "email": email.message_from_string(data[0][1]),
6356 "raw_message": data[0][1]}
6357 fr["multipart"] = fr["email"].is_multipart()
6358
6359 fr["flags"] = self.driver.ParseFlags(data[1])
6360 fetch_results.append(fr)
6361 else:
6362
6363 raise Exception("IMAP error retrieving the body: %s" % data)
6364 else:
6365 raise Exception("IMAP search error: %s" % search_result[1])
6366 elif isinstance(query, (Expression, basestring)):
6367 raise NotImplementedError()
6368 else:
6369 raise TypeError("Unexpected query type")
6370
6371 imapqry_dict = {}
6372 imapfields_dict = {}
6373
6374 if len(fields) == 1 and isinstance(fields[0], SQLALL):
6375 allfields = True
6376 elif len(fields) == 0:
6377 allfields = True
6378 else:
6379 allfields = False
6380 if allfields:
6381 colnames = ["%s.%s" % (tablename, field) for field in self.search_fields.keys()]
6382 else:
6383 colnames = ["%s.%s" % (tablename, field.name) for field in fields]
6384
6385 for k in colnames:
6386 imapfields_dict[k] = k
6387
6388 imapqry_list = list()
6389 imapqry_array = list()
6390 for fr in fetch_results:
6391 attachments = []
6392 content = []
6393 size = 0
6394 n = int(fr["message"])
6395 item_dict = dict()
6396 message = fr["email"]
6397 uid = fr["uid"]
6398 charset = self.get_charset(message)
6399 flags = fr["flags"]
6400 raw_message = fr["raw_message"]
6401
6402
6403
6404
6405
6406
6407
6408
6409
6410
6411 if "%s.id" % tablename in colnames:
6412 item_dict["%s.id" % tablename] = n
6413 if "%s.created" % tablename in colnames:
6414 item_dict["%s.created" % tablename] = self.convert_date(message["Date"])
6415 if "%s.uid" % tablename in colnames:
6416 item_dict["%s.uid" % tablename] = uid
6417 if "%s.sender" % tablename in colnames:
6418
6419
6420
6421 item_dict["%s.sender" % tablename] = message["From"]
6422 if "%s.to" % tablename in colnames:
6423 item_dict["%s.to" % tablename] = message["To"]
6424 if "%s.cc" % tablename in colnames:
6425 if "Cc" in message.keys():
6426 item_dict["%s.cc" % tablename] = message["Cc"]
6427 else:
6428 item_dict["%s.cc" % tablename] = ""
6429 if "%s.bcc" % tablename in colnames:
6430 if "Bcc" in message.keys():
6431 item_dict["%s.bcc" % tablename] = message["Bcc"]
6432 else:
6433 item_dict["%s.bcc" % tablename] = ""
6434 if "%s.deleted" % tablename in colnames:
6435 item_dict["%s.deleted" % tablename] = "\\Deleted" in flags
6436 if "%s.draft" % tablename in colnames:
6437 item_dict["%s.draft" % tablename] = "\\Draft" in flags
6438 if "%s.flagged" % tablename in colnames:
6439 item_dict["%s.flagged" % tablename] = "\\Flagged" in flags
6440 if "%s.recent" % tablename in colnames:
6441 item_dict["%s.recent" % tablename] = "\\Recent" in flags
6442 if "%s.seen" % tablename in colnames:
6443 item_dict["%s.seen" % tablename] = "\\Seen" in flags
6444 if "%s.subject" % tablename in colnames:
6445 item_dict["%s.subject" % tablename] = message["Subject"]
6446 if "%s.answered" % tablename in colnames:
6447 item_dict["%s.answered" % tablename] = "\\Answered" in flags
6448 if "%s.mime" % tablename in colnames:
6449 item_dict["%s.mime" % tablename] = message.get_content_type()
6450 if "%s.encoding" % tablename in colnames:
6451 item_dict["%s.encoding" % tablename] = charset
6452
6453
6454
6455
6456
6457
6458 if "%s.email" % tablename in colnames:
6459
6460 item_dict["%s.email" % tablename] = raw_message
6461
6462
6463
6464
6465
6466
6467 for part in message.walk():
6468 maintype = part.get_content_maintype()
6469 if ("%s.attachments" % tablename in colnames) or \
6470 ("%s.content" % tablename in colnames):
6471 payload = part.get_payload(decode=True)
6472 if payload:
6473 filename = part.get_filename()
6474 values = {"mime": part.get_content_type()}
6475 if ((filename or not "text" in maintype) and
6476 ("%s.attachments" % tablename in colnames)):
6477 values.update({"payload": payload,
6478 "filename": filename,
6479 "encoding": part.get_content_charset(),
6480 "disposition": part["Content-Disposition"]})
6481 attachments.append(values)
6482 elif (("text" in maintype) and
6483 ("%s.content" % tablename in colnames)):
6484 values.update({"text": self.encode_text(payload,
6485 self.get_charset(part))})
6486 content.append(values)
6487
6488 if "%s.size" % tablename in colnames:
6489 if part is not None:
6490 size += len(str(part))
6491 item_dict["%s.content" % tablename] = content
6492 item_dict["%s.attachments" % tablename] = attachments
6493 item_dict["%s.size" % tablename] = size
6494 imapqry_list.append(item_dict)
6495
6496
6497
6498 for item_dict in imapqry_list:
6499 imapqry_array_item = list()
6500 for fieldname in colnames:
6501 imapqry_array_item.append(item_dict[fieldname])
6502 imapqry_array.append(imapqry_array_item)
6503
6504
6505 colnames = colnames
6506 processor = attributes.get('processor',self.parse)
6507 return processor(imapqry_array, fields, colnames)
6508
6509 - def _insert(self, table, fields):
6510 def add_payload(message, obj):
6511 payload = Message()
6512 payload.set_charset(obj.get("encoding", "utf-8"))
6513 mime = obj.get("mime", None)
6514 if mime:
6515 payload.set_type(mime)
6516 if "text" in obj:
6517 payload.set_payload(obj["text"])
6518 elif "payload" in obj:
6519 payload.set_payload(obj["payload"])
6520 if "filename" in obj and obj["filename"]:
6521 payload.add_header("Content-Disposition",
6522 "attachment", filename=obj["filename"])
6523 message.attach(payload)
6524
6525 mailbox = table.mailbox
6526 d = dict(((k.name, v) for k, v in fields))
6527 date_time = (d.get("created", datetime.datetime.now())).timetuple()
6528 if len(d) > 0:
6529 message = d.get("email", None)
6530 attachments = d.get("attachments", [])
6531 content = d.get("content", [])
6532 flags = " ".join(["\\%s" % flag.capitalize() for flag in
6533 ("answered", "deleted", "draft", "flagged",
6534 "recent", "seen") if d.get(flag, False)])
6535 if not message:
6536 from email.message import Message
6537 mime = d.get("mime", None)
6538 charset = d.get("encoding", None)
6539 message = Message()
6540 message["from"] = d.get("sender", "")
6541 message["subject"] = d.get("subject", "")
6542 if mime:
6543 message.set_type(mime)
6544 if charset:
6545 message.set_charset(charset)
6546 for item in ("to", "cc", "bcc"):
6547 value = d.get(item, "")
6548 if isinstance(value, basestring):
6549 message[item] = value
6550 else:
6551 message[item] = ";".join([i for i in
6552 value])
6553 if (not message.is_multipart() and
6554 (not message.get_content_type().startswith(
6555 "multipart"))):
6556 if isinstance(content, basestring):
6557 message.set_payload(content)
6558 elif len(content) > 0:
6559 message.set_payload(content[0]["text"])
6560 else:
6561 [add_payload(message, c) for c in content]
6562 [add_payload(message, a) for a in attachments]
6563 message = message.as_string()
6564 return (mailbox, flags, date_time, message)
6565 else:
6566 raise NotImplementedError("IMAP empty insert is not implemented")
6567
6568 - def insert(self, table, fields):
6569 values = self._insert(table, fields)
6570 result, data = self.connection.append(*values)
6571 if result == "OK":
6572 uid = int(re.findall("\d+", str(data))[-1])
6573 return self.db(table.uid==uid).select(table.id).first().id
6574 else:
6575 raise Exception("IMAP message append failed: %s" % data)
6576
6577 - def _update(self, tablename, query, fields, commit=False):
6578
6579 commands = list()
6580 if use_common_filters(query):
6581 query = self.common_filter(query, [tablename,])
6582 mark = []
6583 unmark = []
6584 if query:
6585 for item in fields:
6586 field = item[0]
6587 name = field.name
6588 value = item[1]
6589 if self.is_flag(name):
6590 flag = self.search_fields[name]
6591 if (value is not None) and (flag != "\\Recent"):
6592 if value:
6593 mark.append(flag)
6594 else:
6595 unmark.append(flag)
6596 result, data = self.connection.select(
6597 self.connection.mailbox_names[tablename])
6598 string_query = "(%s)" % query
6599 result, data = self.connection.search(None, string_query)
6600 store_list = [item.strip() for item in data[0].split()
6601 if item.strip().isdigit()]
6602
6603 for number in store_list:
6604 result = None
6605 if len(mark) > 0:
6606 commands.append((number, "+FLAGS", "(%s)" % " ".join(mark)))
6607 if len(unmark) > 0:
6608 commands.append((number, "-FLAGS", "(%s)" % " ".join(unmark)))
6609 return commands
6610
6611 - def update(self, tablename, query, fields):
6612 rowcount = 0
6613 commands = self._update(tablename, query, fields)
6614 for command in commands:
6615 result, data = self.connection.store(*command)
6616 if result == "OK":
6617 rowcount += 1
6618 else:
6619 raise Exception("IMAP storing error: %s" % data)
6620 return rowcount
6621
6622 - def _count(self, query, distinct=None):
6623 raise NotImplementedError()
6624
6625 - def count(self,query,distinct=None):
6637
6638 - def delete(self, tablename, query):
6639 counter = 0
6640 if query:
6641 if use_common_filters(query):
6642 query = self.common_filter(query, [tablename,])
6643 result, data = self.connection.select(self.connection.mailbox_names[tablename])
6644 string_query = "(%s)" % query
6645 result, data = self.connection.search(None, string_query)
6646 store_list = [item.strip() for item in data[0].split() if item.strip().isdigit()]
6647 for number in store_list:
6648 result, data = self.connection.store(number, "+FLAGS", "(\\Deleted)")
6649 if result == "OK":
6650 counter += 1
6651 else:
6652 raise Exception("IMAP store error: %s" % data)
6653 if counter > 0:
6654 result, data = self.connection.expunge()
6655 return counter
6656
6657 - def BELONGS(self, first, second):
6658 result = None
6659 name = self.search_fields[first.name]
6660 if name == "MESSAGE":
6661 values = [str(val) for val in second if str(val).isdigit()]
6662 result = "%s" % ",".join(values).strip()
6663
6664 elif name == "UID":
6665 values = [str(val) for val in second if str(val).isdigit()]
6666 result = "UID %s" % ",".join(values).strip()
6667
6668 else:
6669 raise Exception("Operation not supported")
6670
6671 return result
6672
6673 - def CONTAINS(self, first, second, case_sensitive=False):
6674
6675 result = None
6676 name = self.search_fields[first.name]
6677
6678 if name in ("FROM", "TO", "SUBJECT", "TEXT"):
6679 result = "%s \"%s\"" % (name, self.expand(second))
6680 else:
6681 if first.name in ("cc", "bcc"):
6682 result = "%s \"%s\"" % (first.name.upper(), self.expand(second))
6683 elif first.name == "mime":
6684 result = "HEADER Content-Type \"%s\"" % self.expand(second)
6685 else:
6686 raise Exception("Operation not supported")
6687 return result
6688
6689 - def GT(self, first, second):
6690 result = None
6691 name = self.search_fields[first.name]
6692 if name == "MESSAGE":
6693 last_message = self.get_last_message(first.tablename)
6694 result = "%d:%d" % (int(self.expand(second)) + 1, last_message)
6695 elif name == "UID":
6696
6697
6698
6699 try:
6700 pedestal, threshold = self.get_uid_bounds(first.tablename)
6701 except TypeError:
6702 e = sys.exc_info()[1]
6703 LOGGER.debug("Error requesting uid bounds: %s", str(e))
6704 return ""
6705 try:
6706 lower_limit = int(self.expand(second)) + 1
6707 except (ValueError, TypeError):
6708 e = sys.exc_info()[1]
6709 raise Exception("Operation not supported (non integer UID)")
6710 result = "UID %s:%s" % (lower_limit, threshold)
6711 elif name == "DATE":
6712 result = "SINCE %s" % self.convert_date(second, add=datetime.timedelta(1))
6713 elif name == "SIZE":
6714 result = "LARGER %s" % self.expand(second)
6715 else:
6716 raise Exception("Operation not supported")
6717 return result
6718
6719 - def GE(self, first, second):
6720 result = None
6721 name = self.search_fields[first.name]
6722 if name == "MESSAGE":
6723 last_message = self.get_last_message(first.tablename)
6724 result = "%s:%s" % (self.expand(second), last_message)
6725 elif name == "UID":
6726
6727
6728
6729 try:
6730 pedestal, threshold = self.get_uid_bounds(first.tablename)
6731 except TypeError:
6732 e = sys.exc_info()[1]
6733 LOGGER.debug("Error requesting uid bounds: %s", str(e))
6734 return ""
6735 lower_limit = self.expand(second)
6736 result = "UID %s:%s" % (lower_limit, threshold)
6737 elif name == "DATE":
6738 result = "SINCE %s" % self.convert_date(second)
6739 else:
6740 raise Exception("Operation not supported")
6741 return result
6742
6743 - def LT(self, first, second):
6744 result = None
6745 name = self.search_fields[first.name]
6746 if name == "MESSAGE":
6747 result = "%s:%s" % (1, int(self.expand(second)) - 1)
6748 elif name == "UID":
6749 try:
6750 pedestal, threshold = self.get_uid_bounds(first.tablename)
6751 except TypeError:
6752 e = sys.exc_info()[1]
6753 LOGGER.debug("Error requesting uid bounds: %s", str(e))
6754 return ""
6755 try:
6756 upper_limit = int(self.expand(second)) - 1
6757 except (ValueError, TypeError):
6758 e = sys.exc_info()[1]
6759 raise Exception("Operation not supported (non integer UID)")
6760 result = "UID %s:%s" % (pedestal, upper_limit)
6761 elif name == "DATE":
6762 result = "BEFORE %s" % self.convert_date(second)
6763 elif name == "SIZE":
6764 result = "SMALLER %s" % self.expand(second)
6765 else:
6766 raise Exception("Operation not supported")
6767 return result
6768
6769 - def LE(self, first, second):
6770 result = None
6771 name = self.search_fields[first.name]
6772 if name == "MESSAGE":
6773 result = "%s:%s" % (1, self.expand(second))
6774 elif name == "UID":
6775 try:
6776 pedestal, threshold = self.get_uid_bounds(first.tablename)
6777 except TypeError:
6778 e = sys.exc_info()[1]
6779 LOGGER.debug("Error requesting uid bounds: %s", str(e))
6780 return ""
6781 upper_limit = int(self.expand(second))
6782 result = "UID %s:%s" % (pedestal, upper_limit)
6783 elif name == "DATE":
6784 result = "BEFORE %s" % self.convert_date(second, add=datetime.timedelta(1))
6785 else:
6786 raise Exception("Operation not supported")
6787 return result
6788
6789 - def NE(self, first, second=None):
6790 if (second is None) and isinstance(first, Field):
6791
6792 if first.type == "id":
6793 return self.GE(first, 1)
6794 result = self.NOT(self.EQ(first, second))
6795 result = result.replace("NOT NOT", "").strip()
6796 return result
6797
6798 - def EQ(self,first,second):
6799 name = self.search_fields[first.name]
6800 result = None
6801 if name is not None:
6802 if name == "MESSAGE":
6803
6804 result = "%s" % self.expand(second)
6805 elif name == "UID":
6806 result = "UID %s" % self.expand(second)
6807 elif name == "DATE":
6808 result = "ON %s" % self.convert_date(second)
6809
6810 elif name in self.flags.values():
6811 if second:
6812 result = "%s" % (name.upper()[1:])
6813 else:
6814 result = "NOT %s" % (name.upper()[1:])
6815 else:
6816 raise Exception("Operation not supported")
6817 else:
6818 raise Exception("Operation not supported")
6819 return result
6820
6821 - def AND(self, first, second):
6824
6825 - def OR(self, first, second):
6828
6829 - def NOT(self, first):
6830 result = "NOT %s" % self.expand(first)
6831 return result
6832
6833
6834
6835
6836
6837 ADAPTERS = {
6838 'sqlite': SQLiteAdapter,
6839 'spatialite': SpatiaLiteAdapter,
6840 'sqlite:memory': SQLiteAdapter,
6841 'spatialite:memory': SpatiaLiteAdapter,
6842 'mysql': MySQLAdapter,
6843 'postgres': PostgreSQLAdapter,
6844 'postgres:psycopg2': PostgreSQLAdapter,
6845 'postgres:pg8000': PostgreSQLAdapter,
6846 'postgres2:psycopg2': NewPostgreSQLAdapter,
6847 'postgres2:pg8000': NewPostgreSQLAdapter,
6848 'oracle': OracleAdapter,
6849 'mssql': MSSQLAdapter,
6850 'mssql2': MSSQL2Adapter,
6851 'mssql3': MSSQL3Adapter,
6852 'vertica': VerticaAdapter,
6853 'sybase': SybaseAdapter,
6854 'db2': DB2Adapter,
6855 'teradata': TeradataAdapter,
6856 'informix': InformixAdapter,
6857 'informix-se': InformixSEAdapter,
6858 'firebird': FireBirdAdapter,
6859 'firebird_embedded': FireBirdAdapter,
6860 'ingres': IngresAdapter,
6861 'ingresu': IngresUnicodeAdapter,
6862 'sapdb': SAPDBAdapter,
6863 'cubrid': CubridAdapter,
6864 'jdbc:sqlite': JDBCSQLiteAdapter,
6865 'jdbc:sqlite:memory': JDBCSQLiteAdapter,
6866 'jdbc:postgres': JDBCPostgreSQLAdapter,
6867 'gae': GoogleDatastoreAdapter,
6868 'google:datastore': GoogleDatastoreAdapter,
6869 'google:sql': GoogleSQLAdapter,
6870 'couchdb': CouchDBAdapter,
6871 'mongodb': MongoDBAdapter,
6872 'imap': IMAPAdapter
6873 }
6876 """
6877 Field type validation, using web2py's validators mechanism.
6878
6879 makes sure the content of a field is in line with the declared
6880 fieldtype
6881 """
6882 db = field.db
6883 try:
6884 from gluon import validators
6885 except ImportError:
6886 return []
6887 field_type, field_length = field.type, field.length
6888 if isinstance(field_type, SQLCustomType):
6889 if hasattr(field_type, 'validator'):
6890 return field_type.validator
6891 else:
6892 field_type = field_type.type
6893 elif not isinstance(field_type,str):
6894 return []
6895 requires=[]
6896 def ff(r,id):
6897 row=r(id)
6898 if not row:
6899 return id
6900 elif hasattr(r, '_format') and isinstance(r._format,str):
6901 return r._format % row
6902 elif hasattr(r, '_format') and callable(r._format):
6903 return r._format(row)
6904 else:
6905 return id
6906 if field_type in (('string', 'text', 'password')):
6907 requires.append(validators.IS_LENGTH(field_length))
6908 elif field_type == 'json':
6909 requires.append(validators.IS_EMPTY_OR(validators.IS_JSON(native_json=field.db._adapter.native_json)))
6910 elif field_type == 'double' or field_type == 'float':
6911 requires.append(validators.IS_FLOAT_IN_RANGE(-1e100, 1e100))
6912 elif field_type in ('integer','bigint'):
6913 requires.append(validators.IS_INT_IN_RANGE(-1e100, 1e100))
6914 elif field_type.startswith('decimal'):
6915 requires.append(validators.IS_DECIMAL_IN_RANGE(-10**10, 10**10))
6916 elif field_type == 'date':
6917 requires.append(validators.IS_DATE())
6918 elif field_type == 'time':
6919 requires.append(validators.IS_TIME())
6920 elif field_type == 'datetime':
6921 requires.append(validators.IS_DATETIME())
6922 elif db and field_type.startswith('reference') and \
6923 field_type.find('.') < 0 and \
6924 field_type[10:] in db.tables:
6925 referenced = db[field_type[10:]]
6926 def repr_ref(id, row=None, r=referenced, f=ff): return f(r, id)
6927 field.represent = field.represent or repr_ref
6928 if hasattr(referenced, '_format') and referenced._format:
6929 requires = validators.IS_IN_DB(db,referenced._id,
6930 referenced._format)
6931 if field.unique:
6932 requires._and = validators.IS_NOT_IN_DB(db,field)
6933 if field.tablename == field_type[10:]:
6934 return validators.IS_EMPTY_OR(requires)
6935 return requires
6936 elif db and field_type.startswith('list:reference') and \
6937 field_type.find('.') < 0 and \
6938 field_type[15:] in db.tables:
6939 referenced = db[field_type[15:]]
6940 def list_ref_repr(ids, row=None, r=referenced, f=ff):
6941 if not ids:
6942 return None
6943 refs = None
6944 db, id = r._db, r._id
6945 if isinstance(db._adapter, GoogleDatastoreAdapter):
6946 def count(values): return db(id.belongs(values)).select(id)
6947 rx = range(0, len(ids), 30)
6948 refs = reduce(lambda a,b:a&b, [count(ids[i:i+30]) for i in rx])
6949 else:
6950 refs = db(id.belongs(ids)).select(id)
6951 return (refs and ', '.join(f(r,x.id) for x in refs) or '')
6952 field.represent = field.represent or list_ref_repr
6953 if hasattr(referenced, '_format') and referenced._format:
6954 requires = validators.IS_IN_DB(db,referenced._id,
6955 referenced._format,multiple=True)
6956 else:
6957 requires = validators.IS_IN_DB(db,referenced._id,
6958 multiple=True)
6959 if field.unique:
6960 requires._and = validators.IS_NOT_IN_DB(db,field)
6961 if not field.notnull:
6962 requires = validators.IS_EMPTY_OR(requires)
6963 return requires
6964 elif field_type.startswith('list:'):
6965 def repr_list(values,row=None): return', '.join(str(v) for v in (values or []))
6966 field.represent = field.represent or repr_list
6967 if field.unique:
6968 requires.insert(0,validators.IS_NOT_IN_DB(db,field))
6969 sff = ['in', 'do', 'da', 'ti', 'de', 'bo']
6970 if field.notnull and not field_type[:2] in sff:
6971 requires.insert(0, validators.IS_NOT_EMPTY())
6972 elif not field.notnull and field_type[:2] in sff and requires:
6973 requires[-1] = validators.IS_EMPTY_OR(requires[-1])
6974 return requires
6975
6978 return str(item).replace('|', '||')
6979
6982
6984 if not hasattr(value,'split') and hasattr(value,'read'):
6985 value = value.read()
6986 return [long(x) for x in value.split('|') if x.strip()]
6987
6991
6992
6993 -class Row(object):
6994
6995 """
6996 a dictionary that lets you do d['a'] as well as d.a
6997 this is only used to store a Row
6998 """
6999
7000 __init__ = lambda self,*args,**kwargs: self.__dict__.update(*args,**kwargs)
7001
7003 key=str(k)
7004 _extra = self.__dict__.get('_extra', None)
7005 if _extra is not None:
7006 v = _extra.get(key, DEFAULT)
7007 if v != DEFAULT:
7008 return v
7009 m = REGEX_TABLE_DOT_FIELD.match(key)
7010 if m:
7011 try:
7012 return ogetattr(self, m.group(1))[m.group(2)]
7013 except (KeyError,AttributeError,TypeError):
7014 key = m.group(2)
7015 try:
7016 return ogetattr(self, key)
7017 except (KeyError,AttributeError,TypeError), ae:
7018 try:
7019 self[key] = ogetattr(self,'__get_lazy_reference__')(key)
7020 return self[key]
7021 except:
7022 raise ae
7023
7024 __setitem__ = lambda self, key, value: setattr(self, str(key), value)
7025
7026 __delitem__ = object.__delattr__
7027
7028 __copy__ = lambda self: Row(self)
7029
7030 __call__ = __getitem__
7031
7032
7033 - def get(self, key, default=None):
7034 try:
7035 return self.__getitem__(key)
7036 except(KeyError, AttributeError, TypeError):
7037 return self.__dict__.get(key,default)
7038
7039 has_key = __contains__ = lambda self, key: key in self.__dict__
7040
7041 __nonzero__ = lambda self: len(self.__dict__)>0
7042
7043 update = lambda self, *args, **kwargs: self.__dict__.update(*args, **kwargs)
7044
7045 keys = lambda self: self.__dict__.keys()
7046
7047 items = lambda self: self.__dict__.items()
7048
7049 values = lambda self: self.__dict__.values()
7050
7051 __iter__ = lambda self: self.__dict__.__iter__()
7052
7053 iteritems = lambda self: self.__dict__.iteritems()
7054
7055 __str__ = __repr__ = lambda self: '<Row %s>' % self.as_dict()
7056
7057 __int__ = lambda self: object.__getattribute__(self,'id')
7058
7059 __long__ = lambda self: long(object.__getattribute__(self,'id'))
7060
7061 __getattr__ = __getitem__
7062
7063
7064
7065
7066
7067
7068
7069
7070
7071
7073 try:
7074 return self.as_dict() == other.as_dict()
7075 except AttributeError:
7076 return False
7077
7079 return not (self == other)
7080
7082 return Row(dict(self))
7083
7084 - def as_dict(self, datetime_to_str=False, custom_types=None):
7085 SERIALIZABLE_TYPES = [str, unicode, int, long, float, bool, list, dict]
7086 if isinstance(custom_types,(list,tuple,set)):
7087 SERIALIZABLE_TYPES += custom_types
7088 elif custom_types:
7089 SERIALIZABLE_TYPES.append(custom_types)
7090 d = dict(self)
7091 for k in copy.copy(d.keys()):
7092 v=d[k]
7093 if d[k] is None:
7094 continue
7095 elif isinstance(v,Row):
7096 d[k]=v.as_dict()
7097 elif isinstance(v,Reference):
7098 d[k]=long(v)
7099 elif isinstance(v,decimal.Decimal):
7100 d[k]=float(v)
7101 elif isinstance(v, (datetime.date, datetime.datetime, datetime.time)):
7102 if datetime_to_str:
7103 d[k] = v.isoformat().replace('T',' ')[:19]
7104 elif not isinstance(v,tuple(SERIALIZABLE_TYPES)):
7105 del d[k]
7106 return d
7107
7108 - def as_xml(self, row_name="row", colnames=None, indent=' '):
7109 def f(row,field,indent=' '):
7110 if isinstance(row,Row):
7111 spc = indent+' \n'
7112 items = [f(row[x],x,indent+' ') for x in row]
7113 return '%s<%s>\n%s\n%s</%s>' % (
7114 indent,
7115 field,
7116 spc.join(item for item in items if item),
7117 indent,
7118 field)
7119 elif not callable(row):
7120 if REGEX_ALPHANUMERIC.match(field):
7121 return '%s<%s>%s</%s>' % (indent,field,row,field)
7122 else:
7123 return '%s<extra name="%s">%s</extra>' % \
7124 (indent,field,row)
7125 else:
7126 return None
7127 return f(self, row_name, indent=indent)
7128
7129 - def as_json(self, mode="object", default=None, colnames=None,
7130 serialize=True, **kwargs):
7131 """
7132 serializes the row to a JSON object
7133 kwargs are passed to .as_dict method
7134 only "object" mode supported
7135
7136 serialize = False used by Rows.as_json
7137 TODO: return array mode with query column order
7138
7139 mode and colnames are not implemented
7140 """
7141
7142 item = self.as_dict(**kwargs)
7143 if serialize:
7144 if have_serializers:
7145 return serializers.json(item,
7146 default=default or
7147 serializers.custom_json)
7148 elif simplejson:
7149 return simplejson.dumps(item)
7150 else:
7151 raise RuntimeError("missing simplejson")
7152 else:
7153 return item
7154
7164
7166 if not isinstance(fields,(list,tuple)):
7167 fields = [fields]
7168 new_fields = []
7169 for field in fields:
7170 if isinstance(field,Field):
7171 new_fields.append(field)
7172 elif isinstance(field,Table):
7173 for ofield in field:
7174 new_fields.append(ofield)
7175 else:
7176 raise RuntimeError("fields must be a list of fields")
7177 fields = new_fields
7178 field_map = {}
7179 for field in fields:
7180 n = field.name.lower()
7181 if not n in field_map:
7182 field_map[n] = field
7183 n = str(field).lower()
7184 if not n in field_map:
7185 field_map[n] = field
7186 constants = {}
7187 i = 0
7188 while True:
7189 m = REGEX_CONST_STRING.search(text)
7190 if not m: break
7191 text = text[:m.start()]+('#%i' % i)+text[m.end():]
7192 constants[str(i)] = m.group()[1:-1]
7193 i+=1
7194 text = re.sub('\s+',' ',text).lower()
7195 for a,b in [('&','and'),
7196 ('|','or'),
7197 ('~','not'),
7198 ('==','='),
7199 ('<','<'),
7200 ('>','>'),
7201 ('<=','<='),
7202 ('>=','>='),
7203 ('<>','!='),
7204 ('=<','<='),
7205 ('=>','>='),
7206 ('=','='),
7207 (' less or equal than ','<='),
7208 (' greater or equal than ','>='),
7209 (' equal or less than ','<='),
7210 (' equal or greater than ','>='),
7211 (' less or equal ','<='),
7212 (' greater or equal ','>='),
7213 (' equal or less ','<='),
7214 (' equal or greater ','>='),
7215 (' not equal to ','!='),
7216 (' not equal ','!='),
7217 (' equal to ','='),
7218 (' equal ','='),
7219 (' equals ','='),
7220 (' less than ','<'),
7221 (' greater than ','>'),
7222 (' starts with ','startswith'),
7223 (' ends with ','endswith'),
7224 (' not in ' , 'notbelongs'),
7225 (' in ' , 'belongs'),
7226 (' is ','=')]:
7227 if a[0]==' ':
7228 text = text.replace(' is'+a,' %s ' % b)
7229 text = text.replace(a,' %s ' % b)
7230 text = re.sub('\s+',' ',text).lower()
7231 text = re.sub('(?P<a>[\<\>\!\=])\s+(?P<b>[\<\>\!\=])','\g<a>\g<b>',text)
7232 query = field = neg = op = logic = None
7233 for item in text.split():
7234 if field is None:
7235 if item == 'not':
7236 neg = True
7237 elif not neg and not logic and item in ('and','or'):
7238 logic = item
7239 elif item in field_map:
7240 field = field_map[item]
7241 else:
7242 raise RuntimeError("Invalid syntax")
7243 elif not field is None and op is None:
7244 op = item
7245 elif not op is None:
7246 if item.startswith('#'):
7247 if not item[1:] in constants:
7248 raise RuntimeError("Invalid syntax")
7249 value = constants[item[1:]]
7250 else:
7251 value = item
7252 if field.type in ('text', 'string', 'json'):
7253 if op == '=': op = 'like'
7254 if op == '=': new_query = field==value
7255 elif op == '<': new_query = field<value
7256 elif op == '>': new_query = field>value
7257 elif op == '<=': new_query = field<=value
7258 elif op == '>=': new_query = field>=value
7259 elif op == '!=': new_query = field!=value
7260 elif op == 'belongs': new_query = field.belongs(value.split(','))
7261 elif op == 'notbelongs': new_query = ~field.belongs(value.split(','))
7262 elif field.type in ('text', 'string', 'json'):
7263 if op == 'contains': new_query = field.contains(value)
7264 elif op == 'like': new_query = field.like(value)
7265 elif op == 'startswith': new_query = field.startswith(value)
7266 elif op == 'endswith': new_query = field.endswith(value)
7267 else: raise RuntimeError("Invalid operation")
7268 elif field._db._adapter.dbengine=='google:datastore' and \
7269 field.type in ('list:integer', 'list:string', 'list:reference'):
7270 if op == 'contains': new_query = field.contains(value)
7271 else: raise RuntimeError("Invalid operation")
7272 else: raise RuntimeError("Invalid operation")
7273 if neg: new_query = ~new_query
7274 if query is None:
7275 query = new_query
7276 elif logic == 'and':
7277 query &= new_query
7278 elif logic == 'or':
7279 query |= new_query
7280 field = op = neg = logic = None
7281 return query
7282
7284
7285 """
7286 an instance of this class represents a database connection
7287
7288 Example::
7289
7290 db = DAL('sqlite://test.db')
7291
7292 or
7293
7294 db = DAL(**{"uri": ..., "tables": [...]...}) # experimental
7295
7296 db.define_table('tablename', Field('fieldname1'),
7297 Field('fieldname2'))
7298 """
7299
7300 - def __new__(cls, uri='sqlite://dummy.db', *args, **kwargs):
7327
7328 @staticmethod
7330 """
7331 # ## this allows gluon to set a folder for this thread
7332 # ## <<<<<<<<< Should go away as new DAL replaces old sql.py
7333 """
7334 BaseAdapter.set_folder(folder)
7335
7336 @staticmethod
7338 """
7339 Returns a dictionary with uri as key with timings and defined tables
7340 {'sqlite://storage.sqlite': {
7341 'dbstats': [(select auth_user.email from auth_user, 0.02009)],
7342 'dbtables': {
7343 'defined': ['auth_cas', 'auth_event', 'auth_group',
7344 'auth_membership', 'auth_permission', 'auth_user'],
7345 'lazy': '[]'
7346 }
7347 }
7348 }
7349 """
7350 dbs = getattr(THREAD_LOCAL,'db_instances',{}).items()
7351 infos = {}
7352 for db_uid, db_group in dbs:
7353 for db in db_group:
7354 if not db._uri:
7355 continue
7356 k = hide_password(db._uri)
7357 infos[k] = dict(dbstats = [(row[0], row[1]) for row in db._timings],
7358 dbtables = {'defined':
7359 sorted(list(set(db.tables) -
7360 set(db._LAZY_TABLES.keys()))),
7361 'lazy': sorted(db._LAZY_TABLES.keys())}
7362 )
7363 return infos
7364
7365 @staticmethod
7378
7379 @staticmethod
7401
7402 - def __init__(self, uri=DEFAULT_URI,
7403 pool_size=0, folder=None,
7404 db_codec='UTF-8', check_reserved=None,
7405 migrate=True, fake_migrate=False,
7406 migrate_enabled=True, fake_migrate_all=False,
7407 decode_credentials=False, driver_args=None,
7408 adapter_args=None, attempts=5, auto_import=False,
7409 bigint_id=False, debug=False, lazy_tables=False,
7410 db_uid=None, do_connect=True,
7411 after_connection=None, tables=None):
7412 """
7413 Creates a new Database Abstraction Layer instance.
7414
7415 Keyword arguments:
7416
7417 :uri: string that contains information for connecting to a database.
7418 (default: 'sqlite://dummy.db')
7419
7420 experimental: you can specify a dictionary as uri
7421 parameter i.e. with
7422 db = DAL({"uri": "sqlite://storage.sqlite",
7423 "tables": {...}, ...})
7424
7425 for an example of dict input you can check the output
7426 of the scaffolding db model with
7427
7428 db.as_dict()
7429
7430 Note that for compatibility with Python older than
7431 version 2.6.5 you should cast your dict input keys
7432 to str due to a syntax limitation on kwarg names.
7433 for proper DAL dictionary input you can use one of:
7434
7435 obj = serializers.cast_keys(dict, [encoding="utf-8"])
7436
7437 or else (for parsing json input)
7438
7439 obj = serializers.loads_json(data, unicode_keys=False)
7440
7441 :pool_size: How many open connections to make to the database object.
7442 :folder: where .table files will be created.
7443 automatically set within web2py
7444 use an explicit path when using DAL outside web2py
7445 :db_codec: string encoding of the database (default: 'UTF-8')
7446 :check_reserved: list of adapters to check tablenames and column names
7447 against sql/nosql reserved keywords. (Default None)
7448
7449 * 'common' List of sql keywords that are common to all database types
7450 such as "SELECT, INSERT". (recommended)
7451 * 'all' Checks against all known SQL keywords. (not recommended)
7452 <adaptername> Checks against the specific adapters list of keywords
7453 (recommended)
7454 * '<adaptername>_nonreserved' Checks against the specific adapters
7455 list of nonreserved keywords. (if available)
7456 :migrate (defaults to True) sets default migrate behavior for all tables
7457 :fake_migrate (defaults to False) sets default fake_migrate behavior for all tables
7458 :migrate_enabled (defaults to True). If set to False disables ALL migrations
7459 :fake_migrate_all (defaults to False). If sets to True fake migrates ALL tables
7460 :attempts (defaults to 5). Number of times to attempt connecting
7461 :auto_import (defaults to False). If set, import automatically table definitions from the
7462 databases folder
7463 :bigint_id (defaults to False): If set, turn on bigint instead of int for id fields
7464 :lazy_tables (defaults to False): delay table definition until table access
7465 :after_connection (defaults to None): a callable that will be execute after the connection
7466 """
7467 if uri == '<zombie>' and db_uid is not None: return
7468 if not decode_credentials:
7469 credential_decoder = lambda cred: cred
7470 else:
7471 credential_decoder = lambda cred: urllib.unquote(cred)
7472 self._folder = folder
7473 if folder:
7474 self.set_folder(folder)
7475 self._uri = uri
7476 self._pool_size = pool_size
7477 self._db_codec = db_codec
7478 self._lastsql = ''
7479 self._timings = []
7480 self._pending_references = {}
7481 self._request_tenant = 'request_tenant'
7482 self._common_fields = []
7483 self._referee_name = '%(table)s'
7484 self._bigint_id = bigint_id
7485 self._debug = debug
7486 self._migrated = []
7487 self._LAZY_TABLES = {}
7488 self._lazy_tables = lazy_tables
7489 self._tables = SQLCallableList()
7490 self._driver_args = driver_args
7491 self._adapter_args = adapter_args
7492 self._check_reserved = check_reserved
7493 self._decode_credentials = decode_credentials
7494 self._attempts = attempts
7495 self._do_connect = do_connect
7496
7497 if not str(attempts).isdigit() or attempts < 0:
7498 attempts = 5
7499 if uri:
7500 uris = isinstance(uri,(list,tuple)) and uri or [uri]
7501 error = ''
7502 connected = False
7503 for k in range(attempts):
7504 for uri in uris:
7505 try:
7506 if is_jdbc and not uri.startswith('jdbc:'):
7507 uri = 'jdbc:'+uri
7508 self._dbname = REGEX_DBNAME.match(uri).group()
7509 if not self._dbname in ADAPTERS:
7510 raise SyntaxError("Error in URI '%s' or database not supported" % self._dbname)
7511
7512
7513 kwargs = dict(db=self,uri=uri,
7514 pool_size=pool_size,
7515 folder=folder,
7516 db_codec=db_codec,
7517 credential_decoder=credential_decoder,
7518 driver_args=driver_args or {},
7519 adapter_args=adapter_args or {},
7520 do_connect=do_connect,
7521 after_connection=after_connection)
7522 self._adapter = ADAPTERS[self._dbname](**kwargs)
7523 types = ADAPTERS[self._dbname].types
7524
7525 self._adapter.types = copy.copy(types)
7526 self._adapter.build_parsemap()
7527 if bigint_id:
7528 if 'big-id' in types and 'reference' in types:
7529 self._adapter.types['id'] = types['big-id']
7530 self._adapter.types['reference'] = types['big-reference']
7531 connected = True
7532 break
7533 except SyntaxError:
7534 raise
7535 except Exception:
7536 tb = traceback.format_exc()
7537 sys.stderr.write('DEBUG: connect attempt %i, connection error:\n%s' % (k, tb))
7538 if connected:
7539 break
7540 else:
7541 time.sleep(1)
7542 if not connected:
7543 raise RuntimeError("Failure to connect, tried %d times:\n%s" % (attempts, tb))
7544 else:
7545 self._adapter = BaseAdapter(db=self,pool_size=0,
7546 uri='None',folder=folder,
7547 db_codec=db_codec, after_connection=after_connection)
7548 migrate = fake_migrate = False
7549 adapter = self._adapter
7550 self._uri_hash = hashlib_md5(adapter.uri).hexdigest()
7551 self.check_reserved = check_reserved
7552 if self.check_reserved:
7553 from reserved_sql_keywords import ADAPTERS as RSK
7554 self.RSK = RSK
7555 self._migrate = migrate
7556 self._fake_migrate = fake_migrate
7557 self._migrate_enabled = migrate_enabled
7558 self._fake_migrate_all = fake_migrate_all
7559 if auto_import or tables:
7560 self.import_table_definitions(adapter.folder,
7561 tables=tables)
7562
7563 @property
7566
7569 pattern = pjoin(path,self._uri_hash+'_*.table')
7570 if tables:
7571 for table in tables:
7572 self.define_table(**table)
7573 else:
7574 for filename in glob.glob(pattern):
7575 tfile = self._adapter.file_open(filename, 'r')
7576 try:
7577 sql_fields = pickle.load(tfile)
7578 name = filename[len(pattern)-7:-6]
7579 mf = [(value['sortable'],
7580 Field(key,
7581 type=value['type'],
7582 length=value.get('length',None),
7583 notnull=value.get('notnull',False),
7584 unique=value.get('unique',False))) \
7585 for key, value in sql_fields.iteritems()]
7586 mf.sort(lambda a,b: cmp(a[0],b[0]))
7587 self.define_table(name,*[item[1] for item in mf],
7588 **dict(migrate=migrate,
7589 fake_migrate=fake_migrate))
7590 finally:
7591 self._adapter.file_close(tfile)
7592
7594 """
7595 Validates ``name`` against SQL keywords
7596 Uses self.check_reserve which is a list of
7597 operators to use.
7598 self.check_reserved
7599 ['common', 'postgres', 'mysql']
7600 self.check_reserved
7601 ['all']
7602 """
7603 for backend in self.check_reserved:
7604 if name.upper() in self.RSK[backend]:
7605 raise SyntaxError(
7606 'invalid table/column name "%s" is a "%s" reserved SQL/NOSQL keyword' % (name, backend.upper()))
7607
7608 - def parse_as_rest(self,patterns,args,vars,queries=None,nested_select=True):
7609 """
7610 EXAMPLE:
7611
7612 db.define_table('person',Field('name'),Field('info'))
7613 db.define_table('pet',Field('ownedby',db.person),Field('name'),Field('info'))
7614
7615 @request.restful()
7616 def index():
7617 def GET(*args,**vars):
7618 patterns = [
7619 "/friends[person]",
7620 "/{person.name}/:field",
7621 "/{person.name}/pets[pet.ownedby]",
7622 "/{person.name}/pets[pet.ownedby]/{pet.name}",
7623 "/{person.name}/pets[pet.ownedby]/{pet.name}/:field",
7624 ("/dogs[pet]", db.pet.info=='dog'),
7625 ("/dogs[pet]/{pet.name.startswith}", db.pet.info=='dog'),
7626 ]
7627 parser = db.parse_as_rest(patterns,args,vars)
7628 if parser.status == 200:
7629 return dict(content=parser.response)
7630 else:
7631 raise HTTP(parser.status,parser.error)
7632
7633 def POST(table_name,**vars):
7634 if table_name == 'person':
7635 return db.person.validate_and_insert(**vars)
7636 elif table_name == 'pet':
7637 return db.pet.validate_and_insert(**vars)
7638 else:
7639 raise HTTP(400)
7640 return locals()
7641 """
7642
7643 db = self
7644 re1 = REGEX_SEARCH_PATTERN
7645 re2 = REGEX_SQUARE_BRACKETS
7646
7647 def auto_table(table,base='',depth=0):
7648 patterns = []
7649 for field in db[table].fields:
7650 if base:
7651 tag = '%s/%s' % (base,field.replace('_','-'))
7652 else:
7653 tag = '/%s/%s' % (table.replace('_','-'),field.replace('_','-'))
7654 f = db[table][field]
7655 if not f.readable: continue
7656 if f.type=='id' or 'slug' in field or f.type.startswith('reference'):
7657 tag += '/{%s.%s}' % (table,field)
7658 patterns.append(tag)
7659 patterns.append(tag+'/:field')
7660 elif f.type.startswith('boolean'):
7661 tag += '/{%s.%s}' % (table,field)
7662 patterns.append(tag)
7663 patterns.append(tag+'/:field')
7664 elif f.type in ('float','double','integer','bigint'):
7665 tag += '/{%s.%s.ge}/{%s.%s.lt}' % (table,field,table,field)
7666 patterns.append(tag)
7667 patterns.append(tag+'/:field')
7668 elif f.type.startswith('list:'):
7669 tag += '/{%s.%s.contains}' % (table,field)
7670 patterns.append(tag)
7671 patterns.append(tag+'/:field')
7672 elif f.type in ('date','datetime'):
7673 tag+= '/{%s.%s.year}' % (table,field)
7674 patterns.append(tag)
7675 patterns.append(tag+'/:field')
7676 tag+='/{%s.%s.month}' % (table,field)
7677 patterns.append(tag)
7678 patterns.append(tag+'/:field')
7679 tag+='/{%s.%s.day}' % (table,field)
7680 patterns.append(tag)
7681 patterns.append(tag+'/:field')
7682 if f.type in ('datetime','time'):
7683 tag+= '/{%s.%s.hour}' % (table,field)
7684 patterns.append(tag)
7685 patterns.append(tag+'/:field')
7686 tag+='/{%s.%s.minute}' % (table,field)
7687 patterns.append(tag)
7688 patterns.append(tag+'/:field')
7689 tag+='/{%s.%s.second}' % (table,field)
7690 patterns.append(tag)
7691 patterns.append(tag+'/:field')
7692 if depth>0:
7693 for f in db[table]._referenced_by:
7694 tag+='/%s[%s.%s]' % (table,f.tablename,f.name)
7695 patterns.append(tag)
7696 patterns += auto_table(table,base=tag,depth=depth-1)
7697 return patterns
7698
7699 if patterns == 'auto':
7700 patterns=[]
7701 for table in db.tables:
7702 if not table.startswith('auth_'):
7703 patterns.append('/%s[%s]' % (table,table))
7704 patterns += auto_table(table,base='',depth=1)
7705 else:
7706 i = 0
7707 while i<len(patterns):
7708 pattern = patterns[i]
7709 if not isinstance(pattern,str):
7710 pattern = pattern[0]
7711 tokens = pattern.split('/')
7712 if tokens[-1].startswith(':auto') and re2.match(tokens[-1]):
7713 new_patterns = auto_table(tokens[-1][tokens[-1].find('[')+1:-1],
7714 '/'.join(tokens[:-1]))
7715 patterns = patterns[:i]+new_patterns+patterns[i+1:]
7716 i += len(new_patterns)
7717 else:
7718 i += 1
7719 if '/'.join(args) == 'patterns':
7720 return Row({'status':200,'pattern':'list',
7721 'error':None,'response':patterns})
7722 for pattern in patterns:
7723 basequery, exposedfields = None, []
7724 if isinstance(pattern,tuple):
7725 if len(pattern)==2:
7726 pattern, basequery = pattern
7727 elif len(pattern)>2:
7728 pattern, basequery, exposedfields = pattern[0:3]
7729 otable=table=None
7730 if not isinstance(queries,dict):
7731 dbset=db(queries)
7732 if basequery is not None:
7733 dbset = dbset(basequery)
7734 i=0
7735 tags = pattern[1:].split('/')
7736 if len(tags)!=len(args):
7737 continue
7738 for tag in tags:
7739 if re1.match(tag):
7740
7741 tokens = tag[1:-1].split('.')
7742 table, field = tokens[0], tokens[1]
7743 if not otable or table == otable:
7744 if len(tokens)==2 or tokens[2]=='eq':
7745 query = db[table][field]==args[i]
7746 elif tokens[2]=='ne':
7747 query = db[table][field]!=args[i]
7748 elif tokens[2]=='lt':
7749 query = db[table][field]<args[i]
7750 elif tokens[2]=='gt':
7751 query = db[table][field]>args[i]
7752 elif tokens[2]=='ge':
7753 query = db[table][field]>=args[i]
7754 elif tokens[2]=='le':
7755 query = db[table][field]<=args[i]
7756 elif tokens[2]=='year':
7757 query = db[table][field].year()==args[i]
7758 elif tokens[2]=='month':
7759 query = db[table][field].month()==args[i]
7760 elif tokens[2]=='day':
7761 query = db[table][field].day()==args[i]
7762 elif tokens[2]=='hour':
7763 query = db[table][field].hour()==args[i]
7764 elif tokens[2]=='minute':
7765 query = db[table][field].minutes()==args[i]
7766 elif tokens[2]=='second':
7767 query = db[table][field].seconds()==args[i]
7768 elif tokens[2]=='startswith':
7769 query = db[table][field].startswith(args[i])
7770 elif tokens[2]=='contains':
7771 query = db[table][field].contains(args[i])
7772 else:
7773 raise RuntimeError("invalid pattern: %s" % pattern)
7774 if len(tokens)==4 and tokens[3]=='not':
7775 query = ~query
7776 elif len(tokens)>=4:
7777 raise RuntimeError("invalid pattern: %s" % pattern)
7778 if not otable and isinstance(queries,dict):
7779 dbset = db(queries[table])
7780 if basequery is not None:
7781 dbset = dbset(basequery)
7782 dbset=dbset(query)
7783 else:
7784 raise RuntimeError("missing relation in pattern: %s" % pattern)
7785 elif re2.match(tag) and args[i]==tag[:tag.find('[')]:
7786 ref = tag[tag.find('[')+1:-1]
7787 if '.' in ref and otable:
7788 table,field = ref.split('.')
7789 selfld = '_id'
7790 if db[table][field].type.startswith('reference '):
7791 refs = [ x.name for x in db[otable] if x.type == db[table][field].type ]
7792 else:
7793 refs = [ x.name for x in db[table]._referenced_by if x.tablename==otable ]
7794 if refs:
7795 selfld = refs[0]
7796 if nested_select:
7797 try:
7798 dbset=db(db[table][field].belongs(dbset._select(db[otable][selfld])))
7799 except ValueError:
7800 return Row({'status':400,'pattern':pattern,
7801 'error':'invalid path','response':None})
7802 else:
7803 items = [item.id for item in dbset.select(db[otable][selfld])]
7804 dbset=db(db[table][field].belongs(items))
7805 else:
7806 table = ref
7807 if not otable and isinstance(queries,dict):
7808 dbset = db(queries[table])
7809 dbset=dbset(db[table])
7810 elif tag==':field' and table:
7811
7812 field = args[i]
7813 if not field in db[table]: break
7814
7815 if not db[table][field].readable:
7816 return Row({'status':418,'pattern':pattern,
7817 'error':'I\'m a teapot','response':None})
7818 try:
7819 distinct = vars.get('distinct', False) == 'True'
7820 offset = long(vars.get('offset',None) or 0)
7821 limits = (offset,long(vars.get('limit',None) or 1000)+offset)
7822 except ValueError:
7823 return Row({'status':400,'error':'invalid limits','response':None})
7824 items = dbset.select(db[table][field], distinct=distinct, limitby=limits)
7825 if items:
7826 return Row({'status':200,'response':items,
7827 'pattern':pattern})
7828 else:
7829 return Row({'status':404,'pattern':pattern,
7830 'error':'no record found','response':None})
7831 elif tag != args[i]:
7832 break
7833 otable = table
7834 i += 1
7835 if i==len(tags) and table:
7836 ofields = vars.get('order',db[table]._id.name).split('|')
7837 try:
7838 orderby = [db[table][f] if not f.startswith('~') else ~db[table][f[1:]] for f in ofields]
7839 except (KeyError, AttributeError):
7840 return Row({'status':400,'error':'invalid orderby','response':None})
7841 if exposedfields:
7842 fields = [field for field in db[table] if str(field).split('.')[-1] in exposedfields and field.readable]
7843 else:
7844 fields = [field for field in db[table] if field.readable]
7845 count = dbset.count()
7846 try:
7847 offset = long(vars.get('offset',None) or 0)
7848 limits = (offset,long(vars.get('limit',None) or 1000)+offset)
7849 except ValueError:
7850 return Row({'status':400,'error':'invalid limits','response':None})
7851 if count > limits[1]-limits[0]:
7852 return Row({'status':400,'error':'too many records','response':None})
7853 try:
7854 response = dbset.select(limitby=limits,orderby=orderby,*fields)
7855 except ValueError:
7856 return Row({'status':400,'pattern':pattern,
7857 'error':'invalid path','response':None})
7858 return Row({'status':200,'response':response,
7859 'pattern':pattern,'count':count})
7860 return Row({'status':400,'error':'no matching pattern','response':None})
7861
7862 - def define_table(
7863 self,
7864 tablename,
7865 *fields,
7866 **args
7867 ):
7868 if not fields and 'fields' in args:
7869 fields = args.get('fields',())
7870 if not isinstance(tablename, str):
7871 if isinstance(tablename, unicode):
7872 try:
7873 tablename = str(tablename)
7874 except UnicodeEncodeError:
7875 raise SyntaxError("invalid unicode table name")
7876 else:
7877 raise SyntaxError("missing table name")
7878 elif hasattr(self,tablename) or tablename in self.tables:
7879 if not args.get('redefine',False):
7880 raise SyntaxError('table already defined: %s' % tablename)
7881 elif tablename.startswith('_') or hasattr(self,tablename) or \
7882 REGEX_PYTHON_KEYWORDS.match(tablename):
7883 raise SyntaxError('invalid table name: %s' % tablename)
7884 elif self.check_reserved:
7885 self.check_reserved_keyword(tablename)
7886 else:
7887 invalid_args = set(args)-TABLE_ARGS
7888 if invalid_args:
7889 raise SyntaxError('invalid table "%s" attributes: %s' \
7890 % (tablename,invalid_args))
7891 if self._lazy_tables and not tablename in self._LAZY_TABLES:
7892 self._LAZY_TABLES[tablename] = (tablename,fields,args)
7893 table = None
7894 else:
7895 table = self.lazy_define_table(tablename,*fields,**args)
7896 if not tablename in self.tables:
7897 self.tables.append(tablename)
7898 return table
7899
7900 - def lazy_define_table(
7901 self,
7902 tablename,
7903 *fields,
7904 **args
7905 ):
7906 args_get = args.get
7907 common_fields = self._common_fields
7908 if common_fields:
7909 fields = list(fields) + list(common_fields)
7910
7911 table_class = args_get('table_class',Table)
7912 table = table_class(self, tablename, *fields, **args)
7913 table._actual = True
7914 self[tablename] = table
7915
7916 table._create_references()
7917 for field in table:
7918 if field.requires == DEFAULT:
7919 field.requires = sqlhtml_validators(field)
7920
7921 migrate = self._migrate_enabled and args_get('migrate',self._migrate)
7922 if migrate and not self._uri in (None,'None') \
7923 or self._adapter.dbengine=='google:datastore':
7924 fake_migrate = self._fake_migrate_all or \
7925 args_get('fake_migrate',self._fake_migrate)
7926 polymodel = args_get('polymodel',None)
7927 try:
7928 GLOBAL_LOCKER.acquire()
7929 self._lastsql = self._adapter.create_table(
7930 table,migrate=migrate,
7931 fake_migrate=fake_migrate,
7932 polymodel=polymodel)
7933 finally:
7934 GLOBAL_LOCKER.release()
7935 else:
7936 table._dbt = None
7937 on_define = args_get('on_define',None)
7938 if on_define: on_define(table)
7939 return table
7940
7941 - def as_dict(self, flat=False, sanitize=True):
7942 db_uid = uri = None
7943 if not sanitize:
7944 uri, db_uid = (self._uri, self._db_uid)
7945 db_as_dict = dict(tables=[], uri=uri, db_uid=db_uid,
7946 **dict([(k, getattr(self, "_" + k, None))
7947 for k in 'pool_size','folder','db_codec',
7948 'check_reserved','migrate','fake_migrate',
7949 'migrate_enabled','fake_migrate_all',
7950 'decode_credentials','driver_args',
7951 'adapter_args', 'attempts',
7952 'bigint_id','debug','lazy_tables',
7953 'do_connect']))
7954 for table in self:
7955 db_as_dict["tables"].append(table.as_dict(flat=flat,
7956 sanitize=sanitize))
7957 return db_as_dict
7958
7959 - def as_xml(self, sanitize=True):
7964
7965 - def as_json(self, sanitize=True):
7970
7971 - def as_yaml(self, sanitize=True):
7976
7978 try:
7979 return tablename in self.tables
7980 except AttributeError:
7981
7982 return False
7983
7984 has_key = __contains__
7985
7986 - def get(self,key,default=None):
7987 return self.__dict__.get(key,default)
7988
7990 for tablename in self.tables:
7991 yield self[tablename]
7992
7995
7997 if ogetattr(self,'_lazy_tables') and \
7998 key in ogetattr(self,'_LAZY_TABLES'):
7999 tablename, fields, args = self._LAZY_TABLES.pop(key)
8000 return self.lazy_define_table(tablename,*fields,**args)
8001 return ogetattr(self, key)
8002
8004 osetattr(self, str(key), value)
8005
8007 if key[:1]!='_' and key in self:
8008 raise SyntaxError(
8009 'Object %s exists and cannot be redefined' % key)
8010 osetattr(self,key,value)
8011
8012 __delitem__ = object.__delattr__
8013
8015 if hasattr(self,'_uri'):
8016 return '<DAL uri="%s">' % hide_password(str(self._uri))
8017 else:
8018 return '<DAL db_uid="%s">' % self._db_uid
8019
8022
8023 - def __call__(self, query=None, ignore_common_filters=None):
8024 if isinstance(query,Table):
8025 query = self._adapter.id_query(query)
8026 elif isinstance(query,Field):
8027 query = query!=None
8028 elif isinstance(query, dict):
8029 icf = query.get("ignore_common_filters")
8030 if icf: ignore_common_filters = icf
8031 return Set(self, query, ignore_common_filters=ignore_common_filters)
8032
8035
8038
8040 self._adapter.close()
8041 if self._db_uid in THREAD_LOCAL.db_instances:
8042 db_group = THREAD_LOCAL.db_instances[self._db_uid]
8043 db_group.remove(self)
8044 if not db_group:
8045 del THREAD_LOCAL.db_instances[self._db_uid]
8046
8047 - def executesql(self, query, placeholders=None, as_dict=False,
8048 fields=None, colnames=None):
8049 """
8050 placeholders is optional and will always be None.
8051 If using raw SQL with placeholders, placeholders may be
8052 a sequence of values to be substituted in
8053 or, (if supported by the DB driver), a dictionary with keys
8054 matching named placeholders in your SQL.
8055
8056 Added 2009-12-05 "as_dict" optional argument. Will always be
8057 None when using DAL. If using raw SQL can be set to True
8058 and the results cursor returned by the DB driver will be
8059 converted to a sequence of dictionaries keyed with the db
8060 field names. Tested with SQLite but should work with any database
8061 since the cursor.description used to get field names is part of the
8062 Python dbi 2.0 specs. Results returned with as_dict=True are
8063 the same as those returned when applying .to_list() to a DAL query.
8064
8065 [{field1: value1, field2: value2}, {field1: value1b, field2: value2b}]
8066
8067 Added 2012-08-24 "fields" and "colnames" optional arguments. If either
8068 is provided, the results cursor returned by the DB driver will be
8069 converted to a DAL Rows object using the db._adapter.parse() method.
8070
8071 The "fields" argument is a list of DAL Field objects that match the
8072 fields returned from the DB. The Field objects should be part of one or
8073 more Table objects defined on the DAL object. The "fields" list can
8074 include one or more DAL Table objects in addition to or instead of
8075 including Field objects, or it can be just a single table (not in a
8076 list). In that case, the Field objects will be extracted from the
8077 table(s).
8078
8079 Instead of specifying the "fields" argument, the "colnames" argument
8080 can be specified as a list of field names in tablename.fieldname format.
8081 Again, these should represent tables and fields defined on the DAL
8082 object.
8083
8084 It is also possible to specify both "fields" and the associated
8085 "colnames". In that case, "fields" can also include DAL Expression
8086 objects in addition to Field objects. For Field objects in "fields",
8087 the associated "colnames" must still be in tablename.fieldname format.
8088 For Expression objects in "fields", the associated "colnames" can
8089 be any arbitrary labels.
8090
8091 Note, the DAL Table objects referred to by "fields" or "colnames" can
8092 be dummy tables and do not have to represent any real tables in the
8093 database. Also, note that the "fields" and "colnames" must be in the
8094 same order as the fields in the results cursor returned from the DB.
8095 """
8096 adapter = self._adapter
8097 if placeholders:
8098 adapter.execute(query, placeholders)
8099 else:
8100 adapter.execute(query)
8101 if as_dict:
8102 if not hasattr(adapter.cursor,'description'):
8103 raise RuntimeError("database does not support executesql(...,as_dict=True)")
8104
8105
8106
8107 columns = adapter.cursor.description
8108
8109 fields = [f[0] for f in columns]
8110
8111 data = adapter._fetchall()
8112
8113
8114 return [dict(zip(fields,row)) for row in data]
8115 try:
8116 data = adapter._fetchall()
8117 except:
8118 return None
8119 if fields or colnames:
8120 fields = [] if fields is None else fields
8121 if not isinstance(fields, list):
8122 fields = [fields]
8123 extracted_fields = []
8124 for field in fields:
8125 if isinstance(field, Table):
8126 extracted_fields.extend([f for f in field])
8127 else:
8128 extracted_fields.append(field)
8129 if not colnames:
8130 colnames = ['%s.%s' % (f.tablename, f.name)
8131 for f in extracted_fields]
8132 data = adapter.parse(
8133 data, fields=extracted_fields, colnames=colnames)
8134 return data
8135
8137 for table in self:
8138 table._referenced_by = [field for field in table._referenced_by
8139 if not field.table==thistable]
8140
8142 step = long(kwargs.get('max_fetch_rows,',500))
8143 write_colnames = kwargs['write_colnames'] = \
8144 kwargs.get("write_colnames", True)
8145 for table in self.tables:
8146 ofile.write('TABLE %s\r\n' % table)
8147 query = self._adapter.id_query(self[table])
8148 nrows = self(query).count()
8149 kwargs['write_colnames'] = write_colnames
8150 for k in range(0,nrows,step):
8151 self(query).select(limitby=(k,k+step)).export_to_csv_file(
8152 ofile, *args, **kwargs)
8153 kwargs['write_colnames'] = False
8154 ofile.write('\r\n\r\n')
8155 ofile.write('END')
8156
8157 - def import_from_csv_file(self, ifile, id_map=None, null='<NULL>',
8158 unique='uuid', map_tablenames=None,
8159 ignore_missing_tables=False,
8160 *args, **kwargs):
8161
8162 id_offset = {}
8163 map_tablenames = map_tablenames or {}
8164 for line in ifile:
8165 line = line.strip()
8166 if not line:
8167 continue
8168 elif line == 'END':
8169 return
8170 elif not line.startswith('TABLE ') or \
8171 not line[6:] in self.tables:
8172 raise SyntaxError('invalid file format')
8173 else:
8174 tablename = line[6:]
8175 tablename = map_tablenames.get(tablename,tablename)
8176 if tablename is not None and tablename in self.tables:
8177 self[tablename].import_from_csv_file(
8178 ifile, id_map, null, unique, id_offset,
8179 *args, **kwargs)
8180 elif tablename is None or ignore_missing_tables:
8181
8182 for line in ifile:
8183 if not line.strip():
8184 break
8185 else:
8186 raise RuntimeError("Unable to import table that does not exist.\nTry db.import_from_csv_file(..., map_tablenames={'table':'othertable'},ignore_missing_tables=True)")
8187
8190 return DAL('<zombie>',db_uid=db_uid)
8191
8194
8195 copyreg.pickle(DAL, DAL_pickler, DAL_unpickler)
8198 """
8199 Helper class providing a comma-separated string having all the field names
8200 (prefixed by table name and '.')
8201
8202 normally only called from within gluon.sql
8203 """
8204
8207
8209 return ', '.join([str(field) for field in self._table])
8210
8213
8215 if not self._record:
8216 self._record = self._table[long(self)]
8217 if not self._record:
8218 raise RuntimeError(
8219 "Using a recursive select but encountered a broken reference: %s %d"%(self._table, long(self)))
8220
8222 if key == 'id':
8223 return long(self)
8224 if key in self._table:
8225 self.__allocate()
8226 if self._record:
8227 return self._record.get(key,None)
8228 else:
8229 return None
8230
8231 - def get(self, key, default=None):
8233
8240
8242 if key == 'id':
8243 return long(self)
8244 self.__allocate()
8245 return self._record.get(key, None)
8246
8248 self.__allocate()
8249 self._record[key] = value
8250
8253 return marshal.loads(data)
8254
8256 try:
8257 marshal_dump = marshal.dumps(long(data))
8258 except AttributeError:
8259 marshal_dump = 'i%s' % struct.pack('<i', long(data))
8260 return (Reference_unpickler, (marshal_dump,))
8261
8262 copyreg.pickle(Reference, Reference_pickler, Reference_unpickler)
8272 def _decorated(f):
8273 instance = self.table
8274 import types
8275 method = types.MethodType(f, instance, instance.__class__)
8276 name = method_name or f.func_name
8277 setattr(instance, name, method)
8278 return f
8279 return _decorated
8280
8282
8283 """
8284 an instance of this class represents a database table
8285
8286 Example::
8287
8288 db = DAL(...)
8289 db.define_table('users', Field('name'))
8290 db.users.insert(name='me') # print db.users._insert(...) to see SQL
8291 db.users.drop()
8292 """
8293
8294 - def __init__(
8295 self,
8296 db,
8297 tablename,
8298 *fields,
8299 **args
8300 ):
8301 """
8302 Initializes the table and performs checking on the provided fields.
8303
8304 Each table will have automatically an 'id'.
8305
8306 If a field is of type Table, the fields (excluding 'id') from that table
8307 will be used instead.
8308
8309 :raises SyntaxError: when a supplied field is of incorrect type.
8310 """
8311 self._actual = False
8312 self._tablename = tablename
8313 self._ot = args.get('actual_name')
8314 self._sequence_name = args.get('sequence_name') or \
8315 db and db._adapter.sequence_name(tablename)
8316 self._trigger_name = args.get('trigger_name') or \
8317 db and db._adapter.trigger_name(tablename)
8318 self._common_filter = args.get('common_filter')
8319 self._format = args.get('format')
8320 self._singular = args.get(
8321 'singular',tablename.replace('_',' ').capitalize())
8322 self._plural = args.get(
8323 'plural',pluralize(self._singular.lower()).capitalize())
8324
8325 if 'primarykey' in args and args['primarykey'] is not None:
8326 self._primarykey = args.get('primarykey')
8327
8328 self._before_insert = []
8329 self._before_update = [Set.delete_uploaded_files]
8330 self._before_delete = [Set.delete_uploaded_files]
8331 self._after_insert = []
8332 self._after_update = []
8333 self._after_delete = []
8334
8335 self.add_method = MethodAdder(self)
8336
8337 fieldnames,newfields=set(),[]
8338 _primarykey = getattr(self, '_primarykey', None)
8339 if _primarykey is not None:
8340 if not isinstance(_primarykey, list):
8341 raise SyntaxError(
8342 "primarykey must be a list of fields from table '%s'" \
8343 % tablename)
8344 if len(_primarykey)==1:
8345 self._id = [f for f in fields if isinstance(f,Field) \
8346 and f.name==_primarykey[0]][0]
8347 elif not [f for f in fields if (isinstance(f,Field) and
8348 f.type=='id') or (isinstance(f, dict) and
8349 f.get("type", None)=="id")]:
8350 field = Field('id', 'id')
8351 newfields.append(field)
8352 fieldnames.add('id')
8353 self._id = field
8354 virtual_fields = []
8355 def include_new(field):
8356 newfields.append(field)
8357 fieldnames.add(field.name)
8358 if field.type=='id':
8359 self._id = field
8360 for field in fields:
8361 if isinstance(field, (FieldMethod, FieldVirtual)):
8362 virtual_fields.append(field)
8363 elif isinstance(field, Field) and not field.name in fieldnames:
8364 if field.db is not None:
8365 field = copy.copy(field)
8366 include_new(field)
8367 elif isinstance(field, dict) and not field['fieldname'] in fieldnames:
8368 include_new(Field(**field))
8369 elif isinstance(field, Table):
8370 table = field
8371 for field in table:
8372 if not field.name in fieldnames and not field.type=='id':
8373 t2 = not table._actual and self._tablename
8374 include_new(field.clone(point_self_references_to=t2))
8375 elif not isinstance(field, (Field, Table)):
8376 raise SyntaxError(
8377 'define_table argument is not a Field or Table: %s' % field)
8378 fields = newfields
8379 self._db = db
8380 tablename = tablename
8381 self._fields = SQLCallableList()
8382 self.virtualfields = []
8383 fields = list(fields)
8384
8385 if db and db._adapter.uploads_in_blob==True:
8386 uploadfields = [f.name for f in fields if f.type=='blob']
8387 for field in fields:
8388 fn = field.uploadfield
8389 if isinstance(field, Field) and field.type == 'upload'\
8390 and fn is True:
8391 fn = field.uploadfield = '%s_blob' % field.name
8392 if isinstance(fn,str) and not fn in uploadfields:
8393 fields.append(Field(fn,'blob',default='',
8394 writable=False,readable=False))
8395
8396 lower_fieldnames = set()
8397 reserved = dir(Table) + ['fields']
8398 if (db and db.check_reserved):
8399 check_reserved = db.check_reserved_keyword
8400 else:
8401 def check_reserved(field_name):
8402 if field_name in reserved:
8403 raise SyntaxError("field name %s not allowed" % field_name)
8404 for field in fields:
8405 field_name = field.name
8406 check_reserved(field_name)
8407 fn_lower = field_name.lower()
8408 if fn_lower in lower_fieldnames:
8409 raise SyntaxError("duplicate field %s in table %s" \
8410 % (field_name, tablename))
8411 else:
8412 lower_fieldnames.add(fn_lower)
8413
8414 self.fields.append(field_name)
8415 self[field_name] = field
8416 if field.type == 'id':
8417 self['id'] = field
8418 field.tablename = field._tablename = tablename
8419 field.table = field._table = self
8420 field.db = field._db = db
8421 self.ALL = SQLALL(self)
8422
8423 if _primarykey is not None:
8424 for k in _primarykey:
8425 if k not in self.fields:
8426 raise SyntaxError(
8427 "primarykey must be a list of fields from table '%s " % tablename)
8428 else:
8429 self[k].notnull = True
8430 for field in virtual_fields:
8431 self[field.name] = field
8432
8433 @property
8436
8437 - def update(self,*args,**kwargs):
8438 raise RuntimeError("Syntax Not Supported")
8439
8440 - def _enable_record_versioning(self,
8441 archive_db=None,
8442 archive_name = '%(tablename)s_archive',
8443 is_active = 'is_active',
8444 current_record = 'current_record',
8445 current_record_label = None):
8446 db = self._db
8447 archive_db = archive_db or db
8448 archive_name = archive_name % dict(tablename=self._tablename)
8449 if archive_name in archive_db.tables():
8450 return
8451 fieldnames = self.fields()
8452 same_db = archive_db is db
8453 field_type = self if same_db else 'bigint'
8454 clones = []
8455 for field in self:
8456 nfk = same_db or not field.type.startswith('reference')
8457 clones.append(field.clone(
8458 unique=False, type=field.type if nfk else 'bigint'))
8459 archive_db.define_table(
8460 archive_name,
8461 Field(current_record,field_type,label=current_record_label),
8462 *clones,**dict(format=self._format))
8463
8464 self._before_update.append(
8465 lambda qset,fs,db=archive_db,an=archive_name,cn=current_record:
8466 archive_record(qset,fs,db[an],cn))
8467 if is_active and is_active in fieldnames:
8468 self._before_delete.append(
8469 lambda qset: qset.update(is_active=False))
8470 newquery = lambda query, t=self, name=self._tablename: \
8471 reduce(AND,[db[tn].is_active == True
8472 for tn in db._adapter.tables(query)
8473 if tn==name or getattr(db[tn],'_ot',None)==name])
8474 query = self._common_filter
8475 if query:
8476 newquery = query & newquery
8477 self._common_filter = newquery
8478
8486
8488 db = self._db
8489 pr = db._pending_references
8490 self._referenced_by = []
8491 self._references = []
8492 for field in self:
8493 fieldname = field.name
8494 field_type = field.type
8495 if isinstance(field_type,str) and field_type[:10] == 'reference ':
8496 ref = field_type[10:].strip()
8497 if not ref:
8498 SyntaxError('Table: reference to nothing: %s' %ref)
8499 if '.' in ref:
8500 rtablename, throw_it,rfieldname = ref.partition('.')
8501 else:
8502 rtablename, rfieldname = ref, None
8503 if not rtablename in db:
8504 pr[rtablename] = pr.get(rtablename,[]) + [field]
8505 continue
8506 rtable = db[rtablename]
8507 if rfieldname:
8508 if not hasattr(rtable,'_primarykey'):
8509 raise SyntaxError(
8510 'keyed tables can only reference other keyed tables (for now)')
8511 if rfieldname not in rtable.fields:
8512 raise SyntaxError(
8513 "invalid field '%s' for referenced table '%s' in table '%s'" \
8514 % (rfieldname, rtablename, self._tablename))
8515 rfield = rtable[rfieldname]
8516 else:
8517 rfield = rtable._id
8518 rtable._referenced_by.append(field)
8519 field.referent = rfield
8520 self._references.append(field)
8521 else:
8522 field.referent = None
8523 for referee in pr.get(self._tablename,[]):
8524 self._referenced_by.append(referee)
8525
8527 return dict([(k, v) for (k, v) in record.iteritems() if k
8528 in self.fields and (self[k].type!='id' or id)])
8529
8531 """ for keyed table only """
8532 query = None
8533 for k,v in key.iteritems():
8534 if k in self._primarykey:
8535 if query:
8536 query = query & (self[k] == v)
8537 else:
8538 query = (self[k] == v)
8539 else:
8540 raise SyntaxError(
8541 'Field %s is not part of the primary key of %s' % \
8542 (k,self._tablename))
8543 return query
8544
8546 if not key:
8547 return None
8548 elif isinstance(key, dict):
8549 """ for keyed table """
8550 query = self._build_query(key)
8551 return self._db(query).select(limitby=(0,1), orderby_on_limitby=False).first()
8552 elif str(key).isdigit() or 'google' in DRIVERS and isinstance(key, Key):
8553 return self._db(self._id == key).select(limitby=(0,1), orderby_on_limitby=False).first()
8554 elif key:
8555 return ogetattr(self, str(key))
8556
8558 for_update = kwargs.get('_for_update',False)
8559 if '_for_update' in kwargs: del kwargs['_for_update']
8560
8561 orderby = kwargs.get('_orderby',None)
8562 if '_orderby' in kwargs: del kwargs['_orderby']
8563
8564 if not key is DEFAULT:
8565 if isinstance(key, Query):
8566 record = self._db(key).select(
8567 limitby=(0,1),for_update=for_update, orderby=orderby, orderby_on_limitby=False).first()
8568 elif not str(key).isdigit():
8569 record = None
8570 else:
8571 record = self._db(self._id == key).select(
8572 limitby=(0,1),for_update=for_update, orderby=orderby, orderby_on_limitby=False).first()
8573 if record:
8574 for k,v in kwargs.iteritems():
8575 if record[k]!=v: return None
8576 return record
8577 elif kwargs:
8578 query = reduce(lambda a,b:a&b,[self[k]==v for k,v in kwargs.iteritems()])
8579 return self._db(query).select(limitby=(0,1),for_update=for_update, orderby=orderby, orderby_on_limitby=False).first()
8580 else:
8581 return None
8582
8584 if isinstance(key, dict) and isinstance(value, dict):
8585 """ option for keyed table """
8586 if set(key.keys()) == set(self._primarykey):
8587 value = self._filter_fields(value)
8588 kv = {}
8589 kv.update(value)
8590 kv.update(key)
8591 if not self.insert(**kv):
8592 query = self._build_query(key)
8593 self._db(query).update(**self._filter_fields(value))
8594 else:
8595 raise SyntaxError(
8596 'key must have all fields from primary key: %s'%\
8597 (self._primarykey))
8598 elif str(key).isdigit():
8599 if key == 0:
8600 self.insert(**self._filter_fields(value))
8601 elif self._db(self._id == key)\
8602 .update(**self._filter_fields(value)) is None:
8603 raise SyntaxError('No such record: %s' % key)
8604 else:
8605 if isinstance(key, dict):
8606 raise SyntaxError(
8607 'value must be a dictionary: %s' % value)
8608 osetattr(self, str(key), value)
8609
8610 __getattr__ = __getitem__
8611
8613 if key[:1]!='_' and key in self:
8614 raise SyntaxError('Object exists and cannot be redefined: %s' % key)
8615 osetattr(self,key,value)
8616
8618 if isinstance(key, dict):
8619 query = self._build_query(key)
8620 if not self._db(query).delete():
8621 raise SyntaxError('No such record: %s' % key)
8622 elif not str(key).isdigit() or \
8623 not self._db(self._id == key).delete():
8624 raise SyntaxError('No such record: %s' % key)
8625
8627 return hasattr(self,key)
8628
8629 has_key = __contains__
8630
8632 return self.__dict__.items()
8633
8635 for fieldname in self.fields:
8636 yield self[fieldname]
8637
8640
8641
8643 return '<Table %s (%s)>' % (self._tablename,','.join(self.fields()))
8644
8646 if self._ot is not None:
8647 ot = self._db._adapter.QUOTE_TEMPLATE % self._ot
8648 if 'Oracle' in str(type(self._db._adapter)):
8649 return '%s %s' % (ot, self._tablename)
8650 return '%s AS %s' % (ot, self._tablename)
8651 return self._tablename
8652
8653 - def _drop(self, mode = ''):
8654 return self._db._adapter._drop(self, mode)
8655
8656 - def drop(self, mode = ''):
8657 return self._db._adapter.drop(self,mode)
8658
8659 - def _listify(self,fields,update=False):
8660 new_fields = {}
8661
8662
8663 for name in fields:
8664 if not name in self.fields:
8665 if name != 'id':
8666 raise SyntaxError(
8667 'Field %s does not belong to the table' % name)
8668 else:
8669 field = self[name]
8670 value = fields[name]
8671 if field.filter_in:
8672 value = field.filter_in(value)
8673 new_fields[name] = (field,value)
8674
8675
8676 to_compute = []
8677 for ofield in self:
8678 name = ofield.name
8679 if not name in new_fields:
8680
8681 if ofield.compute:
8682 to_compute.append((name,ofield))
8683
8684 elif not update and not ofield.default is None:
8685 value = ofield.default
8686 fields[name] = value
8687 new_fields[name] = (ofield,value)
8688
8689 elif update and not ofield.update is None:
8690 value = ofield.update
8691 fields[name] = value
8692 new_fields[name] = (ofield,value)
8693
8694 elif not update and ofield.required:
8695 raise RuntimeError(
8696 'Table: missing required field: %s' % name)
8697
8698 if to_compute:
8699 row = Row(fields)
8700 for name,ofield in to_compute:
8701
8702 try:
8703 row[name] = new_value = ofield.compute(row)
8704 new_fields[name] = (ofield, new_value)
8705 except (KeyError, AttributeError):
8706
8707 if ofield.required:
8708 raise SyntaxError('unable to compute field: %s' % name)
8709 return new_fields.values()
8710
8712 for field in self:
8713 if field.type=='upload' and field.name in fields:
8714 value = fields[field.name]
8715 if value is not None and not isinstance(value,str):
8716 if hasattr(value,'file') and hasattr(value,'filename'):
8717 new_name = field.store(value.file,filename=value.filename)
8718 elif hasattr(value,'read') and hasattr(value,'name'):
8719 new_name = field.store(value,filename=value.name)
8720 else:
8721 raise RuntimeError("Unable to handle upload")
8722 fields[field.name] = new_name
8723
8725 "If there are no fields/values specified, return table defaults"
8726 if not fields:
8727 fields = {}
8728 for field in self:
8729 if field.type != "id":
8730 fields[field.name] = field.default
8731 return fields
8732
8736
8746
8762
8788
8790 if _key is DEFAULT:
8791 record = self(**values)
8792 elif isinstance(_key,dict):
8793 record = self(**_key)
8794 else:
8795 record = self(_key)
8796 if record:
8797 record.update_record(**values)
8798 newid = None
8799 else:
8800 newid = self.insert(**values)
8801 return newid
8802
8804 """
8805 here items is a list of dictionaries
8806 """
8807 items = [self._listify(item) for item in items]
8808 if any(f(item) for item in items for f in self._before_insert):return 0
8809 ret = self._db._adapter.bulk_insert(self,items)
8810 ret and [[f(item,ret[k]) for k,item in enumerate(items)] for f in self._after_insert]
8811 return ret
8812
8814 return self._db._adapter._truncate(self, mode)
8815
8817 return self._db._adapter.truncate(self, mode)
8818
8819 - def import_from_csv_file(
8820 self,
8821 csvfile,
8822 id_map=None,
8823 null='<NULL>',
8824 unique='uuid',
8825 id_offset=None,
8826 *args, **kwargs
8827 ):
8828 """
8829 Import records from csv file.
8830 Column headers must have same names as table fields.
8831 Field 'id' is ignored.
8832 If column names read 'table.file' the 'table.' prefix is ignored.
8833 'unique' argument is a field which must be unique
8834 (typically a uuid field)
8835 'restore' argument is default False;
8836 if set True will remove old values in table first.
8837 'id_map' ff set to None will not map ids.
8838 The import will keep the id numbers in the restored table.
8839 This assumes that there is an field of type id that
8840 is integer and in incrementing order.
8841 Will keep the id numbers in restored table.
8842 """
8843
8844 delimiter = kwargs.get('delimiter', ',')
8845 quotechar = kwargs.get('quotechar', '"')
8846 quoting = kwargs.get('quoting', csv.QUOTE_MINIMAL)
8847 restore = kwargs.get('restore', False)
8848 if restore:
8849 self._db[self].truncate()
8850
8851 reader = csv.reader(csvfile, delimiter=delimiter,
8852 quotechar=quotechar, quoting=quoting)
8853 colnames = None
8854 if isinstance(id_map, dict):
8855 if not self._tablename in id_map:
8856 id_map[self._tablename] = {}
8857 id_map_self = id_map[self._tablename]
8858
8859 def fix(field, value, id_map, id_offset):
8860 list_reference_s='list:reference'
8861 if value == null:
8862 value = None
8863 elif field.type=='blob':
8864 value = base64.b64decode(value)
8865 elif field.type=='double' or field.type=='float':
8866 if not value.strip():
8867 value = None
8868 else:
8869 value = float(value)
8870 elif field.type in ('integer','bigint'):
8871 if not value.strip():
8872 value = None
8873 else:
8874 value = long(value)
8875 elif field.type.startswith('list:string'):
8876 value = bar_decode_string(value)
8877 elif field.type.startswith(list_reference_s):
8878 ref_table = field.type[len(list_reference_s):].strip()
8879 if id_map is not None:
8880 value = [id_map[ref_table][long(v)] \
8881 for v in bar_decode_string(value)]
8882 else:
8883 value = [v for v in bar_decode_string(value)]
8884 elif field.type.startswith('list:'):
8885 value = bar_decode_integer(value)
8886 elif id_map and field.type.startswith('reference'):
8887 try:
8888 value = id_map[field.type[9:].strip()][long(value)]
8889 except KeyError:
8890 pass
8891 elif id_offset and field.type.startswith('reference'):
8892 try:
8893 value = id_offset[field.type[9:].strip()]+long(value)
8894 except KeyError:
8895 pass
8896 return (field.name, value)
8897
8898 def is_id(colname):
8899 if colname in self:
8900 return self[colname].type == 'id'
8901 else:
8902 return False
8903
8904 first = True
8905 unique_idx = None
8906 for lineno, line in enumerate(reader):
8907 if not line:
8908 break
8909 if not colnames:
8910
8911 colnames = [x.split('.',1)[-1] for x in line][:len(line)]
8912 cols, cid = [], None
8913 for i,colname in enumerate(colnames):
8914 if is_id(colname):
8915 cid = i
8916 elif colname in self.fields:
8917 cols.append((i,self[colname]))
8918 if colname == unique:
8919 unique_idx = i
8920 else:
8921
8922 items = []
8923 for i, field in cols:
8924 try:
8925 items.append(fix(field, line[i], id_map, id_offset))
8926 except ValueError:
8927 raise RuntimeError("Unable to parse line:%s field:%s value:'%s'"
8928 % (lineno+1,field,line[i]))
8929
8930 if not (id_map or cid is None or id_offset is None or unique_idx):
8931 csv_id = long(line[cid])
8932 curr_id = self.insert(**dict(items))
8933 if first:
8934 first = False
8935
8936
8937
8938 id_offset[self._tablename] = (curr_id-csv_id) \
8939 if curr_id>csv_id else 0
8940
8941 while curr_id<csv_id+id_offset[self._tablename]:
8942 self._db(self._db[self][colnames[cid]] == curr_id).delete()
8943 curr_id = self.insert(**dict(items))
8944
8945
8946 elif not unique_idx:
8947 new_id = self.insert(**dict(items))
8948 else:
8949 unique_value = line[unique_idx]
8950 query = self._db[self][unique] == unique_value
8951 record = self._db(query).select().first()
8952 if record:
8953 record.update_record(**dict(items))
8954 new_id = record[self._id.name]
8955 else:
8956 new_id = self.insert(**dict(items))
8957 if id_map and cid is not None:
8958 id_map_self[long(line[cid])] = new_id
8959
8960 - def as_dict(self, flat=False, sanitize=True):
8972
8973 - def as_xml(self, sanitize=True):
8978
8979 - def as_json(self, sanitize=True):
8984
8985 - def as_yaml(self, sanitize=True):
8990
8993
8994 - def on(self, query):
8995 return Expression(self._db,self._db._adapter.ON,self,query)
8996
8998 tablenames = qset.db._adapter.tables(qset.query)
8999 if len(tablenames)!=1: raise RuntimeError("cannot update join")
9000 table = qset.db[tablenames[0]]
9001 for row in qset.select():
9002 fields = archive_table._filter_fields(row)
9003 fields[current_record] = row.id
9004 archive_table.insert(**fields)
9005 return False
9006
9010
9011 - def __init__(
9012 self,
9013 db,
9014 op,
9015 first=None,
9016 second=None,
9017 type=None,
9018 **optional_args
9019 ):
9020
9021 self.db = db
9022 self.op = op
9023 self.first = first
9024 self.second = second
9025 self._table = getattr(first,'_table',None)
9026
9027 if not type and first and hasattr(first,'type'):
9028 self.type = first.type
9029 else:
9030 self.type = type
9031 self.optional_args = optional_args
9032
9036
9040
9044
9048
9052
9056
9060
9064
9068
9072
9076
9080
9084
9088
9092
9096
9100
9104
9106 db = self.db
9107 if start < 0:
9108 pos0 = '(%s - %d)' % (self.len(), abs(start) - 1)
9109 else:
9110 pos0 = start + 1
9111
9112 if stop < 0:
9113 length = '(%s - %d - %s)' % (self.len(), abs(stop) - 1, pos0)
9114 elif stop == sys.maxint:
9115 length = self.len()
9116 else:
9117 length = '(%s - %s)' % (stop + 1, pos0)
9118 return Expression(db,db._adapter.SUBSTRING,
9119 self, (pos0, length), self.type)
9120
9122 return self[i:i + 1]
9123
9125 return self.db._adapter.expand(self,self.type)
9126
9128 db = self.db
9129 return Expression(db,db._adapter.COMMA,self,other,self.type)
9130
9136
9140
9142 db = self.db
9143 if self.type in ('integer','bigint'):
9144 result_type = 'integer'
9145 elif self.type in ['date','time','datetime','double','float']:
9146 result_type = 'double'
9147 elif self.type.startswith('decimal('):
9148 result_type = self.type
9149 else:
9150 raise SyntaxError("subtraction operation not supported for type")
9151 return Expression(db,db._adapter.SUB,self,other,result_type)
9152
9156
9160
9164
9168
9172
9176
9180
9184
9188
9189 - def like(self, value, case_sensitive=False):
9190 db = self.db
9191 op = case_sensitive and db._adapter.LIKE or db._adapter.ILIKE
9192 return Query(db, op, self, value)
9193
9197
9198 - def belongs(self, *value, **kwattr):
9199 """
9200 Accepts the following inputs:
9201 field.belongs(1,2)
9202 field.belongs((1,2))
9203 field.belongs(query)
9204
9205 Does NOT accept:
9206 field.belongs(1)
9207 """
9208 db = self.db
9209 if len(value) == 1:
9210 value = value[0]
9211 if isinstance(value,Query):
9212 value = db(value)._select(value.first._table._id)
9213 elif not isinstance(value, basestring):
9214 value = set(value)
9215 if kwattr.get('null') and None in value:
9216 value.remove(None)
9217 return (self == None) | Query(db, db._adapter.BELONGS, self, value)
9218 return Query(db, db._adapter.BELONGS, self, value)
9219
9221 db = self.db
9222 if not self.type in ('string', 'text', 'json', 'upload'):
9223 raise SyntaxError("startswith used with incompatible field type")
9224 return Query(db, db._adapter.STARTSWITH, self, value)
9225
9227 db = self.db
9228 if not self.type in ('string', 'text', 'json', 'upload'):
9229 raise SyntaxError("endswith used with incompatible field type")
9230 return Query(db, db._adapter.ENDSWITH, self, value)
9231
9232 - def contains(self, value, all=False, case_sensitive=False):
9233 """
9234 The case_sensitive parameters is only useful for PostgreSQL
9235 For other RDMBs it is ignored and contains is always case in-sensitive
9236 For MongoDB and GAE contains is always case sensitive
9237 """
9238 db = self.db
9239 if isinstance(value,(list, tuple)):
9240 subqueries = [self.contains(str(v).strip(),case_sensitive=case_sensitive)
9241 for v in value if str(v).strip()]
9242 if not subqueries:
9243 return self.contains('')
9244 else:
9245 return reduce(all and AND or OR,subqueries)
9246 if not self.type in ('string', 'text', 'json', 'upload') and not self.type.startswith('list:'):
9247 raise SyntaxError("contains used with incompatible field type")
9248 return Query(db, db._adapter.CONTAINS, self, value, case_sensitive=case_sensitive)
9249
9253
9254
9255
9256 - def st_asgeojson(self, precision=15, options=0, version=1):
9260
9261 - def st_astext(self):
9262 db = self.db
9263 return Expression(db, db._adapter.ST_ASTEXT, self, type='string')
9264
9268
9272
9276
9280
9281
9282
9286
9290
9294
9298
9302
9306
9311 """
9312 allows defining of custom SQL types
9313
9314 Example::
9315
9316 decimal = SQLCustomType(
9317 type ='double',
9318 native ='integer',
9319 encoder =(lambda x: int(float(x) * 100)),
9320 decoder = (lambda x: Decimal("0.00") + Decimal(str(float(x)/100)) )
9321 )
9322
9323 db.define_table(
9324 'example',
9325 Field('value', type=decimal)
9326 )
9327
9328 :param type: the web2py type (default = 'string')
9329 :param native: the backend type
9330 :param encoder: how to encode the value to store it in the backend
9331 :param decoder: how to decode the value retrieved from the backend
9332 :param validator: what validators to use ( default = None, will use the
9333 default validator for type)
9334 """
9335
9336 - def __init__(
9337 self,
9338 type='string',
9339 native=None,
9340 encoder=None,
9341 decoder=None,
9342 validator=None,
9343 _class=None,
9344 ):
9345
9346 self.type = type
9347 self.native = native
9348 self.encoder = encoder or (lambda x: x)
9349 self.decoder = decoder or (lambda x: x)
9350 self.validator = validator
9351 self._class = _class or type
9352
9354 try:
9355 return self.type.startswith(self, text)
9356 except TypeError:
9357 return False
9358
9361
9364
9367
9369 - def __init__(self, name, f=None, ftype='string',label=None,table_name=None):
9384 return '%s.%s' % (self.tablename, self.name)
9385
9387 - def __init__(self, name, f=None, handler=None):
9391
9393 return ', '.join(str(y) for y in x or [])
9394
9395 -class Field(Expression):
9396
9397 Virtual = FieldVirtual
9398 Method = FieldMethod
9399 Lazy = FieldMethod
9400
9401 """
9402 an instance of this class represents a database field
9403
9404 example::
9405
9406 a = Field(name, 'string', length=32, default=None, required=False,
9407 requires=IS_NOT_EMPTY(), ondelete='CASCADE',
9408 notnull=False, unique=False,
9409 uploadfield=True, widget=None, label=None, comment=None,
9410 uploadfield=True, # True means store on disk,
9411 # 'a_field_name' means store in this field in db
9412 # False means file content will be discarded.
9413 writable=True, readable=True, update=None, authorize=None,
9414 autodelete=False, represent=None, uploadfolder=None,
9415 uploadseparate=False # upload to separate directories by uuid_keys
9416 # first 2 character and tablename.fieldname
9417 # False - old behavior
9418 # True - put uploaded file in
9419 # <uploaddir>/<tablename>.<fieldname>/uuid_key[:2]
9420 # directory)
9421 uploadfs=None # a pyfilesystem where to store upload
9422
9423 to be used as argument of DAL.define_table
9424
9425 allowed field types:
9426 string, boolean, integer, double, text, blob,
9427 date, time, datetime, upload, password
9428
9429 """
9430
9431 - def __init__(
9432 self,
9433 fieldname,
9434 type='string',
9435 length=None,
9436 default=DEFAULT,
9437 required=False,
9438 requires=DEFAULT,
9439 ondelete='CASCADE',
9440 notnull=False,
9441 unique=False,
9442 uploadfield=True,
9443 widget=None,
9444 label=None,
9445 comment=None,
9446 writable=True,
9447 readable=True,
9448 update=None,
9449 authorize=None,
9450 autodelete=False,
9451 represent=None,
9452 uploadfolder=None,
9453 uploadseparate=False,
9454 uploadfs=None,
9455 compute=None,
9456 custom_store=None,
9457 custom_retrieve=None,
9458 custom_retrieve_file_properties=None,
9459 custom_delete=None,
9460 filter_in = None,
9461 filter_out = None,
9462 custom_qualifier = None,
9463 map_none = None,
9464 ):
9465 self._db = self.db = None
9466 self.op = None
9467 self.first = None
9468 self.second = None
9469 if isinstance(fieldname, unicode):
9470 try:
9471 fieldname = str(fieldname)
9472 except UnicodeEncodeError:
9473 raise SyntaxError('Field: invalid unicode field name')
9474 self.name = fieldname = cleanup(fieldname)
9475 if not isinstance(fieldname, str) or hasattr(Table, fieldname) or \
9476 fieldname[0] == '_' or REGEX_PYTHON_KEYWORDS.match(fieldname):
9477 raise SyntaxError('Field: invalid field name: %s' % fieldname)
9478 self.type = type if not isinstance(type, (Table,Field)) else 'reference %s' % type
9479 self.length = length if not length is None else DEFAULTLENGTH.get(self.type,512)
9480 self.default = default if default!=DEFAULT else (update or None)
9481 self.required = required
9482 self.ondelete = ondelete.upper()
9483 self.notnull = notnull
9484 self.unique = unique
9485 self.uploadfield = uploadfield
9486 self.uploadfolder = uploadfolder
9487 self.uploadseparate = uploadseparate
9488 self.uploadfs = uploadfs
9489 self.widget = widget
9490 self.comment = comment
9491 self.writable = writable
9492 self.readable = readable
9493 self.update = update
9494 self.authorize = authorize
9495 self.autodelete = autodelete
9496 self.represent = list_represent if \
9497 represent==None and type in ('list:integer','list:string') else represent
9498 self.compute = compute
9499 self.isattachment = True
9500 self.custom_store = custom_store
9501 self.custom_retrieve = custom_retrieve
9502 self.custom_retrieve_file_properties = custom_retrieve_file_properties
9503 self.custom_delete = custom_delete
9504 self.filter_in = filter_in
9505 self.filter_out = filter_out
9506 self.custom_qualifier = custom_qualifier
9507 self.label = label if label!=None else fieldname.replace('_',' ').title()
9508 self.requires = requires if requires!=None else []
9509 self.map_none = map_none
9510
9512 self.__dict__.update(*args,**attributes)
9513
9514 - def clone(self,point_self_references_to=False,**args):
9515 field = copy.copy(self)
9516 if point_self_references_to and \
9517 field.type == 'reference %s'+field._tablename:
9518 field.type = 'reference %s' % point_self_references_to
9519 field.__dict__.update(args)
9520 return field
9521
9522 - def store(self, file, filename=None, path=None):
9523 if self.custom_store:
9524 return self.custom_store(file,filename,path)
9525 if isinstance(file, cgi.FieldStorage):
9526 filename = filename or file.filename
9527 file = file.file
9528 elif not filename:
9529 filename = file.name
9530 filename = os.path.basename(filename.replace('/', os.sep)\
9531 .replace('\\', os.sep))
9532 m = REGEX_STORE_PATTERN.search(filename)
9533 extension = m and m.group('e') or 'txt'
9534 uuid_key = web2py_uuid().replace('-', '')[-16:]
9535 encoded_filename = base64.b16encode(filename).lower()
9536 newfilename = '%s.%s.%s.%s' % \
9537 (self._tablename, self.name, uuid_key, encoded_filename)
9538 newfilename = newfilename[:(self.length - 1 - len(extension))] + '.' + extension
9539 self_uploadfield = self.uploadfield
9540 if isinstance(self_uploadfield,Field):
9541 blob_uploadfield_name = self_uploadfield.uploadfield
9542 keys={self_uploadfield.name: newfilename,
9543 blob_uploadfield_name: file.read()}
9544 self_uploadfield.table.insert(**keys)
9545 elif self_uploadfield == True:
9546 if path:
9547 pass
9548 elif self.uploadfolder:
9549 path = self.uploadfolder
9550 elif self.db._adapter.folder:
9551 path = pjoin(self.db._adapter.folder, '..', 'uploads')
9552 else:
9553 raise RuntimeError(
9554 "you must specify a Field(...,uploadfolder=...)")
9555 if self.uploadseparate:
9556 if self.uploadfs:
9557 raise RuntimeError("not supported")
9558 path = pjoin(path,"%s.%s" %(self._tablename, self.name),
9559 uuid_key[:2])
9560 if not exists(path):
9561 os.makedirs(path)
9562 pathfilename = pjoin(path, newfilename)
9563 if self.uploadfs:
9564 dest_file = self.uploadfs.open(newfilename, 'wb')
9565 else:
9566 dest_file = open(pathfilename, 'wb')
9567 try:
9568 shutil.copyfileobj(file, dest_file)
9569 except IOError:
9570 raise IOError(
9571 'Unable to store file "%s" because invalid permissions, readonly file system, or filename too long' % pathfilename)
9572 dest_file.close()
9573 return newfilename
9574
9575 - def retrieve(self, name, path=None, nameonly=False):
9576 """
9577 if nameonly==True return (filename, fullfilename) instead of
9578 (filename, stream)
9579 """
9580 self_uploadfield = self.uploadfield
9581 if self.custom_retrieve:
9582 return self.custom_retrieve(name, path)
9583 import http
9584 if self.authorize or isinstance(self_uploadfield, str):
9585 row = self.db(self == name).select().first()
9586 if not row:
9587 raise http.HTTP(404)
9588 if self.authorize and not self.authorize(row):
9589 raise http.HTTP(403)
9590 file_properties = self.retrieve_file_properties(name,path)
9591 filename = file_properties['filename']
9592 if isinstance(self_uploadfield, str):
9593 stream = StringIO.StringIO(row[self_uploadfield] or '')
9594 elif isinstance(self_uploadfield,Field):
9595 blob_uploadfield_name = self_uploadfield.uploadfield
9596 query = self_uploadfield == name
9597 data = self_uploadfield.table(query)[blob_uploadfield_name]
9598 stream = StringIO.StringIO(data)
9599 elif self.uploadfs:
9600
9601 stream = self.uploadfs.open(name, 'rb')
9602 else:
9603
9604
9605
9606 fullname = pjoin(file_properties['path'],name)
9607 if nameonly:
9608 return (filename, fullname)
9609 stream = open(fullname,'rb')
9610 return (filename, stream)
9611
9613 m = REGEX_UPLOAD_PATTERN.match(name)
9614 if not m or not self.isattachment:
9615 raise TypeError('Can\'t retrieve %s file properties' % name)
9616 self_uploadfield = self.uploadfield
9617 if self.custom_retrieve_file_properties:
9618 return self.custom_retrieve_file_properties(name, path)
9619 if m.group('name'):
9620 try:
9621 filename = base64.b16decode(m.group('name'), True)
9622 filename = REGEX_CLEANUP_FN.sub('_', filename)
9623 except (TypeError, AttributeError):
9624 filename = name
9625 else:
9626 filename = name
9627
9628 if isinstance(self_uploadfield, (str, Field)):
9629 return dict(path=None,filename=filename)
9630
9631 if not path:
9632 if self.uploadfolder:
9633 path = self.uploadfolder
9634 else:
9635 path = pjoin(self.db._adapter.folder, '..', 'uploads')
9636 if self.uploadseparate:
9637 t = m.group('table')
9638 f = m.group('field')
9639 u = m.group('uuidkey')
9640 path = pjoin(path,"%s.%s" % (t,f),u[:2])
9641 return dict(path=path,filename=filename)
9642
9643
9659
9671
9672 - def count(self, distinct=None):
9674
9675 - def as_dict(self, flat=False, sanitize=True):
9676 attrs = ("name", 'authorize', 'represent', 'ondelete',
9677 'custom_store', 'autodelete', 'custom_retrieve',
9678 'filter_out', 'uploadseparate', 'widget', 'uploadfs',
9679 'update', 'custom_delete', 'uploadfield', 'uploadfolder',
9680 'custom_qualifier', 'unique', 'writable', 'compute',
9681 'map_none', 'default', 'type', 'required', 'readable',
9682 'requires', 'comment', 'label', 'length', 'notnull',
9683 'custom_retrieve_file_properties', 'filter_in')
9684 serializable = (int, long, basestring, float, tuple,
9685 bool, type(None))
9686
9687 def flatten(obj):
9688 if isinstance(obj, dict):
9689 return dict((flatten(k), flatten(v)) for k, v in
9690 obj.items())
9691 elif isinstance(obj, (tuple, list, set)):
9692 return [flatten(v) for v in obj]
9693 elif isinstance(obj, serializable):
9694 return obj
9695 elif isinstance(obj, (datetime.datetime,
9696 datetime.date, datetime.time)):
9697 return str(obj)
9698 else:
9699 return None
9700
9701 d = dict()
9702 if not (sanitize and not (self.readable or self.writable)):
9703 for attr in attrs:
9704 if flat:
9705 d.update({attr: flatten(getattr(self, attr))})
9706 else:
9707 d.update({attr: getattr(self, attr)})
9708 d["fieldname"] = d.pop("name")
9709 return d
9710
9711 - def as_xml(self, sanitize=True):
9718
9719 - def as_json(self, sanitize=True):
9726
9727 - def as_yaml(self, sanitize=True):
9733
9736
9738 try:
9739 return '%s.%s' % (self.tablename, self.name)
9740 except:
9741 return '<no table>.%s' % self.name
9742
9743
9744 -class Query(object):
9745
9746 """
9747 a query object necessary to define a set.
9748 it can be stored or can be passed to DAL.__call__() to obtain a Set
9749
9750 Example::
9751
9752 query = db.users.name=='Max'
9753 set = db(query)
9754 records = set.select()
9755
9756 """
9757
9758 - def __init__(
9759 self,
9760 db,
9761 op,
9762 first=None,
9763 second=None,
9764 ignore_common_filters = False,
9765 **optional_args
9766 ):
9767 self.db = self._db = db
9768 self.op = op
9769 self.first = first
9770 self.second = second
9771 self.ignore_common_filters = ignore_common_filters
9772 self.optional_args = optional_args
9773
9776
9778 return self.db._adapter.expand(self)
9779
9782
9783 __rand__ = __and__
9784
9786 return Query(self.db,self.db._adapter.OR,self,other)
9787
9788 __ror__ = __or__
9789
9791 if self.op==self.db._adapter.NOT:
9792 return self.first
9793 return Query(self.db,self.db._adapter.NOT,self)
9794
9796 return repr(self) == repr(other)
9797
9799 return not (self == other)
9800
9801 - def case(self,t=1,f=0):
9802 return self.db._adapter.CASE(self,t,f)
9803
9804 - def as_dict(self, flat=False, sanitize=True):
9805 """Experimental stuff
9806
9807 This allows to return a plain dictionary with the basic
9808 query representation. Can be used with json/xml services
9809 for client-side db I/O
9810
9811 Example:
9812 >>> q = db.auth_user.id != 0
9813 >>> q.as_dict(flat=True)
9814 {"op": "NE", "first":{"tablename": "auth_user",
9815 "fieldname": "id"},
9816 "second":0}
9817 """
9818
9819 SERIALIZABLE_TYPES = (tuple, dict, set, list, int, long, float,
9820 basestring, type(None), bool)
9821 def loop(d):
9822 newd = dict()
9823 for k, v in d.items():
9824 if k in ("first", "second"):
9825 if isinstance(v, self.__class__):
9826 newd[k] = loop(v.__dict__)
9827 elif isinstance(v, Field):
9828 newd[k] = {"tablename": v._tablename,
9829 "fieldname": v.name}
9830 elif isinstance(v, Expression):
9831 newd[k] = loop(v.__dict__)
9832 elif isinstance(v, SERIALIZABLE_TYPES):
9833 newd[k] = v
9834 elif isinstance(v, (datetime.date,
9835 datetime.time,
9836 datetime.datetime)):
9837 newd[k] = unicode(v)
9838 elif k == "op":
9839 if callable(v):
9840 newd[k] = v.__name__
9841 elif isinstance(v, basestring):
9842 newd[k] = v
9843 else: pass
9844 elif isinstance(v, SERIALIZABLE_TYPES):
9845 if isinstance(v, dict):
9846 newd[k] = loop(v)
9847 else: newd[k] = v
9848 return newd
9849
9850 if flat:
9851 return loop(self.__dict__)
9852 else: return self.__dict__
9853
9854
9855 - def as_xml(self, sanitize=True):
9862
9863 - def as_json(self, sanitize=True):
9870
9872 if not orderby:
9873 return None
9874 orderby2 = orderby[0]
9875 for item in orderby[1:]:
9876 orderby2 = orderby2 | item
9877 return orderby2
9878
9880 return (query and hasattr(query,'ignore_common_filters') and \
9881 not query.ignore_common_filters)
9882
9884
9885 """
9886 a Set represents a set of records in the database,
9887 the records are identified by the query=Query(...) object.
9888 normally the Set is generated by DAL.__call__(Query(...))
9889
9890 given a set, for example
9891 set = db(db.users.name=='Max')
9892 you can:
9893 set.update(db.users.name='Massimo')
9894 set.delete() # all elements in the set
9895 set.select(orderby=db.users.id, groupby=db.users.name, limitby=(0,10))
9896 and take subsets:
9897 subset = set(db.users.id<5)
9898 """
9899
9900 - def __init__(self, db, query, ignore_common_filters = None):
9901 self.db = db
9902 self._db = db
9903 self.dquery = None
9904
9905
9906 if isinstance(query, dict):
9907 query = self.parse(query)
9908
9909 if not ignore_common_filters is None and \
9910 use_common_filters(query) == ignore_common_filters:
9911 query = copy.copy(query)
9912 query.ignore_common_filters = ignore_common_filters
9913 self.query = query
9914
9917
9918 - def __call__(self, query, ignore_common_filters=False):
9919 if query is None:
9920 return self
9921 elif isinstance(query,Table):
9922 query = self.db._adapter.id_query(query)
9923 elif isinstance(query,str):
9924 query = Expression(self.db,query)
9925 elif isinstance(query,Field):
9926 query = query!=None
9927 if self.query:
9928 return Set(self.db, self.query & query,
9929 ignore_common_filters=ignore_common_filters)
9930 else:
9931 return Set(self.db, query,
9932 ignore_common_filters=ignore_common_filters)
9933
9934 - def _count(self,distinct=None):
9935 return self.db._adapter._count(self.query,distinct)
9936
9937 - def _select(self, *fields, **attributes):
9938 adapter = self.db._adapter
9939 tablenames = adapter.tables(self.query,
9940 attributes.get('join',None),
9941 attributes.get('left',None),
9942 attributes.get('orderby',None),
9943 attributes.get('groupby',None))
9944 fields = adapter.expand_all(fields, tablenames)
9945 return adapter._select(self.query,fields,attributes)
9946
9948 db = self.db
9949 tablename = db._adapter.get_table(self.query)
9950 return db._adapter._delete(tablename,self.query)
9951
9952 - def _update(self, **update_fields):
9957
9958 - def as_dict(self, flat=False, sanitize=True):
9959 if flat:
9960 uid = dbname = uri = None
9961 codec = self.db._db_codec
9962 if not sanitize:
9963 uri, dbname, uid = (self.db._dbname, str(self.db),
9964 self.db._db_uid)
9965 d = {"query": self.query.as_dict(flat=flat)}
9966 d["db"] = {"uid": uid, "codec": codec,
9967 "name": dbname, "uri": uri}
9968 return d
9969 else: return self.__dict__
9970
9971 - def as_xml(self, sanitize=True):
9978
9979 - def as_json(self, sanitize=True):
9986
9987 - def parse(self, dquery):
9988 "Experimental: Turn a dictionary into a Query object"
9989 self.dquery = dquery
9990 return self.build(self.dquery)
9991
9993 "Experimental: see .parse()"
9994 op, first, second = (d["op"], d["first"],
9995 d.get("second", None))
9996 left = right = built = None
9997
9998 if op in ("AND", "OR"):
9999 if not (type(first), type(second)) == (dict, dict):
10000 raise SyntaxError("Invalid AND/OR query")
10001 if op == "AND":
10002 built = self.build(first) & self.build(second)
10003 else: built = self.build(first) | self.build(second)
10004
10005 elif op == "NOT":
10006 if first is None:
10007 raise SyntaxError("Invalid NOT query")
10008 built = ~self.build(first)
10009 else:
10010
10011 for k, v in {"left": first, "right": second}.items():
10012 if isinstance(v, dict) and v.get("op"):
10013 v = self.build(v)
10014 if isinstance(v, dict) and ("tablename" in v):
10015 v = self.db[v["tablename"]][v["fieldname"]]
10016 if k == "left": left = v
10017 else: right = v
10018
10019 if hasattr(self.db._adapter, op):
10020 opm = getattr(self.db._adapter, op)
10021
10022 if op == "EQ": built = left == right
10023 elif op == "NE": built = left != right
10024 elif op == "GT": built = left > right
10025 elif op == "GE": built = left >= right
10026 elif op == "LT": built = left < right
10027 elif op == "LE": built = left <= right
10028 elif op in ("JOIN", "LEFT_JOIN", "RANDOM", "ALLOW_NULL"):
10029 built = Expression(self.db, opm)
10030 elif op in ("LOWER", "UPPER", "EPOCH", "PRIMARY_KEY",
10031 "COALESCE_ZERO", "RAW", "INVERT"):
10032 built = Expression(self.db, opm, left)
10033 elif op in ("COUNT", "EXTRACT", "AGGREGATE", "SUBSTRING",
10034 "REGEXP", "LIKE", "ILIKE", "STARTSWITH",
10035 "ENDSWITH", "ADD", "SUB", "MUL", "DIV",
10036 "MOD", "AS", "ON", "COMMA", "NOT_NULL",
10037 "COALESCE", "CONTAINS", "BELONGS"):
10038 built = Expression(self.db, opm, left, right)
10039
10040 elif not (left or right): built = Expression(self.db, op)
10041 else:
10042 raise SyntaxError("Operator not supported: %s" % op)
10043
10044 return built
10045
10047 return not self.select(limitby=(0,1), orderby_on_limitby=False)
10048
10049 - def count(self,distinct=None, cache=None):
10050 db = self.db
10051 if cache:
10052 cache_model, time_expire = cache
10053 sql = self._count(distinct=distinct)
10054 key = db._uri + '/' + sql
10055 if len(key)>200: key = hashlib_md5(key).hexdigest()
10056 return cache_model(
10057 key,
10058 (lambda self=self,distinct=distinct: \
10059 db._adapter.count(self.query,distinct)),
10060 time_expire)
10061 return db._adapter.count(self.query,distinct)
10062
10063 - def select(self, *fields, **attributes):
10064 adapter = self.db._adapter
10065 tablenames = adapter.tables(self.query,
10066 attributes.get('join',None),
10067 attributes.get('left',None),
10068 attributes.get('orderby',None),
10069 attributes.get('groupby',None))
10070 fields = adapter.expand_all(fields, tablenames)
10071 return adapter.select(self.query,fields,attributes)
10072
10075
10077 db = self.db
10078 tablename = db._adapter.get_table(self.query)
10079 table = db[tablename]
10080 if any(f(self) for f in table._before_delete): return 0
10081 ret = db._adapter.delete(tablename,self.query)
10082 ret and [f(self) for f in table._after_delete]
10083 return ret
10084
10085 - def update(self, **update_fields):
10086 db = self.db
10087 tablename = db._adapter.get_table(self.query)
10088 table = db[tablename]
10089 table._attempt_upload(update_fields)
10090 if any(f(self,update_fields) for f in table._before_update):
10091 return 0
10092 fields = table._listify(update_fields,update=True)
10093 if not fields:
10094 raise SyntaxError("No fields to update")
10095 ret = db._adapter.update("%s" % table,self.query,fields)
10096 ret and [f(self,update_fields) for f in table._after_update]
10097 return ret
10098
10100 """
10101 same as update but does not call table._before_update and _after_update
10102 """
10103 tablename = self.db._adapter.get_table(self.query)
10104 table = self.db[tablename]
10105 fields = table._listify(update_fields,update=True)
10106 if not fields: raise SyntaxError("No fields to update")
10107
10108 ret = self.db._adapter.update("%s" % table,self.query,fields)
10109 return ret
10110
10112 tablename = self.db._adapter.get_table(self.query)
10113 response = Row()
10114 response.errors = Row()
10115 new_fields = copy.copy(update_fields)
10116 for key,value in update_fields.iteritems():
10117 value,error = self.db[tablename][key].validate(value)
10118 if error:
10119 response.errors[key] = error
10120 else:
10121 new_fields[key] = value
10122 table = self.db[tablename]
10123 if response.errors:
10124 response.updated = None
10125 else:
10126 if not any(f(self,new_fields) for f in table._before_update):
10127 fields = table._listify(new_fields,update=True)
10128 if not fields: raise SyntaxError("No fields to update")
10129 ret = self.db._adapter.update(tablename,self.query,fields)
10130 ret and [f(self,new_fields) for f in table._after_update]
10131 else:
10132 ret = 0
10133 response.updated = ret
10134 return response
10135
10137 table = self.db[self.db._adapter.tables(self.query)[0]]
10138
10139 if upload_fields:
10140 fields = upload_fields.keys()
10141 else:
10142 fields = table.fields
10143 fields = [f for f in fields if table[f].type == 'upload'
10144 and table[f].uploadfield == True
10145 and table[f].autodelete]
10146 if not fields:
10147 return False
10148 for record in self.select(*[table[f] for f in fields]):
10149 for fieldname in fields:
10150 field = table[fieldname]
10151 oldname = record.get(fieldname, None)
10152 if not oldname:
10153 continue
10154 if upload_fields and oldname == upload_fields[fieldname]:
10155 continue
10156 if field.custom_delete:
10157 field.custom_delete(oldname)
10158 else:
10159 uploadfolder = field.uploadfolder
10160 if not uploadfolder:
10161 uploadfolder = pjoin(
10162 self.db._adapter.folder, '..', 'uploads')
10163 if field.uploadseparate:
10164 items = oldname.split('.')
10165 uploadfolder = pjoin(
10166 uploadfolder,
10167 "%s.%s" % (items[0], items[1]),
10168 items[2][:2])
10169 oldpath = pjoin(uploadfolder, oldname)
10170 if exists(oldpath):
10171 os.unlink(oldpath)
10172 return False
10173
10175 - def __init__(self, colset, table, id):
10176 self.colset, self.db, self.tablename, self.id = \
10177 colset, table._db, table._tablename, id
10178
10180 colset, db, tablename, id = self.colset, self.db, self.tablename, self.id
10181 table = db[tablename]
10182 newfields = fields or dict(colset)
10183 for fieldname in newfields.keys():
10184 if not fieldname in table.fields or table[fieldname].type=='id':
10185 del newfields[fieldname]
10186 table._db(table._id==id,ignore_common_filters=True).update(**newfields)
10187 colset.update(newfields)
10188 return colset
10189
10192 self.db, self.tablename, self.id = table._db, table._tablename, id
10194 return self.db(self.db[self.tablename]._id==self.id).delete()
10195
10198 self.db, self.tablename, self.id = table._db, table._tablename, id
10200 if self.db._lazy_tables is False:
10201 raise AttributeError()
10202 table = self.db[self.tablename]
10203 other_table = self.db[other_tablename]
10204 for rfield in table._referenced_by:
10205 if rfield.table == other_table:
10206 return LazySet(rfield, self.id)
10207
10208 raise AttributeError()
10209
10212 self.db, self.tablename, self.fieldname, self.id = \
10213 field.db, field._tablename, field.name, id
10215 query = self.db[self.tablename][self.fieldname]==self.id
10216 return Set(self.db,query)
10219 - def __call__(self, query, ignore_common_filters=False):
10220 return self._getset()(query, ignore_common_filters)
10221 - def _count(self,distinct=None):
10223 - def _select(self, *fields, **attributes):
10227 - def _update(self, **update_fields):
10231 - def count(self,distinct=None, cache=None):
10233 - def select(self, *fields, **attributes):
10239 - def update(self, **update_fields):
10247
10250 self.method=method
10251 self.row=row
10253 return self.method(self.row,*args,**kwargs)
10254
10256 f.__lazy__ = True
10257 return f
10258
10259 -class Rows(object):
10260
10261 """
10262 A wrapper for the return value of a select. It basically represents a table.
10263 It has an iterator and each row is represented as a dictionary.
10264 """
10265
10266
10267
10268 - def __init__(
10269 self,
10270 db=None,
10271 records=[],
10272 colnames=[],
10273 compact=True,
10274 rawrows=None
10275 ):
10276 self.db = db
10277 self.records = records
10278 self.colnames = colnames
10279 self.compact = compact
10280 self.response = rawrows
10281
10283 return '<Rows (%s)>' % len(self.records)
10284
10286 """
10287 db.define_table('x',Field('number','integer'))
10288 if db(db.x).isempty(): [db.x.insert(number=i) for i in range(10)]
10289
10290 from gluon.dal import lazy_virtualfield
10291
10292 class MyVirtualFields(object):
10293 # normal virtual field (backward compatible, discouraged)
10294 def normal_shift(self): return self.x.number+1
10295 # lazy virtual field (because of @staticmethod)
10296 @lazy_virtualfield
10297 def lazy_shift(instance,row,delta=4): return row.x.number+delta
10298 db.x.virtualfields.append(MyVirtualFields())
10299
10300 for row in db(db.x).select():
10301 print row.number, row.normal_shift, row.lazy_shift(delta=7)
10302 """
10303 if not keyed_virtualfields:
10304 return self
10305 for row in self.records:
10306 for (tablename,virtualfields) in keyed_virtualfields.iteritems():
10307 attributes = dir(virtualfields)
10308 if not tablename in row:
10309 box = row[tablename] = Row()
10310 else:
10311 box = row[tablename]
10312 updated = False
10313 for attribute in attributes:
10314 if attribute[0] != '_':
10315 method = getattr(virtualfields,attribute)
10316 if hasattr(method,'__lazy__'):
10317 box[attribute]=VirtualCommand(method,row)
10318 elif type(method)==types.MethodType:
10319 if not updated:
10320 virtualfields.__dict__.update(row)
10321 updated = True
10322 box[attribute]=method()
10323 return self
10324
10326 if self.colnames!=other.colnames:
10327 raise Exception('Cannot & incompatible Rows objects')
10328 records = self.records+other.records
10329 return Rows(self.db,records,self.colnames)
10330
10332 if self.colnames!=other.colnames:
10333 raise Exception('Cannot | incompatible Rows objects')
10334 records = self.records
10335 records += [record for record in other.records \
10336 if not record in records]
10337 return Rows(self.db,records,self.colnames)
10338
10340 if len(self.records):
10341 return 1
10342 return 0
10343
10345 return len(self.records)
10346
10348 return Rows(self.db,self.records[a:b],self.colnames,compact=self.compact)
10349
10351 row = self.records[i]
10352 keys = row.keys()
10353 if self.compact and len(keys) == 1 and keys[0] != '_extra':
10354 return row[row.keys()[0]]
10355 return row
10356
10358 """
10359 iterator over records
10360 """
10361
10362 for i in xrange(len(self)):
10363 yield self[i]
10364
10366 """
10367 serializes the table into a csv file
10368 """
10369
10370 s = StringIO.StringIO()
10371 self.export_to_csv_file(s)
10372 return s.getvalue()
10373
10375 if not self.records:
10376 return None
10377 return self[0]
10378
10380 if not self.records:
10381 return None
10382 return self[-1]
10383
10384 - def find(self,f,limitby=None):
10385 """
10386 returns a new Rows object, a subset of the original object,
10387 filtered by the function f
10388 """
10389 if not self:
10390 return Rows(self.db, [], self.colnames)
10391 records = []
10392 if limitby:
10393 a,b = limitby
10394 else:
10395 a,b = 0,len(self)
10396 k = 0
10397 for row in self:
10398 if f(row):
10399 if a<=k: records.append(row)
10400 k += 1
10401 if k==b: break
10402 return Rows(self.db, records, self.colnames)
10403
10405 """
10406 removes elements from the calling Rows object, filtered by the function f,
10407 and returns a new Rows object containing the removed elements
10408 """
10409 if not self.records:
10410 return Rows(self.db, [], self.colnames)
10411 removed = []
10412 i=0
10413 while i<len(self):
10414 row = self[i]
10415 if f(row):
10416 removed.append(self.records[i])
10417 del self.records[i]
10418 else:
10419 i += 1
10420 return Rows(self.db, removed, self.colnames)
10421
10422 - def sort(self, f, reverse=False):
10423 """
10424 returns a list of sorted elements (not sorted in place)
10425 """
10426 rows = Rows(self.db,[],self.colnames,compact=False)
10427 rows.records = sorted(self,key=f,reverse=reverse)
10428 return rows
10429
10431 """
10432 regroups the rows, by one of the fields
10433 """
10434 one_result = False
10435 if 'one_result' in args:
10436 one_result = args['one_result']
10437
10438 def build_fields_struct(row, fields, num, groups):
10439 ''' helper function:
10440 '''
10441 if num > len(fields)-1:
10442 if one_result:
10443 return row
10444 else:
10445 return [row]
10446
10447 key = fields[num]
10448 value = row[key]
10449
10450 if value not in groups:
10451 groups[value] = build_fields_struct(row, fields, num+1, {})
10452 else:
10453 struct = build_fields_struct(row, fields, num+1, groups[ value ])
10454
10455
10456 if type(struct) == type(dict()):
10457 groups[value].update()
10458
10459 elif type(struct) == type(list()):
10460 groups[value] += struct
10461
10462 else:
10463 groups[value] = struct
10464
10465 return groups
10466
10467 if len(fields) == 0:
10468 return self
10469
10470
10471 if not self.records:
10472 return {}
10473
10474 grouped_row_group = dict()
10475
10476
10477 for row in self:
10478 build_fields_struct(row, fields, 0, grouped_row_group)
10479
10480 return grouped_row_group
10481
10482 - def render(self, i=None, fields=None):
10483 """
10484 Takes an index and returns a copy of the indexed row with values
10485 transformed via the "represent" attributes of the associated fields.
10486
10487 If no index is specified, a generator is returned for iteration
10488 over all the rows.
10489
10490 fields -- a list of fields to transform (if None, all fields with
10491 "represent" attributes will be transformed).
10492 """
10493
10494
10495 if i is None:
10496 return (self.repr(i, fields=fields) for i in range(len(self)))
10497 import sqlhtml
10498 row = copy.deepcopy(self.records[i])
10499 keys = row.keys()
10500 tables = [f.tablename for f in fields] if fields \
10501 else [k for k in keys if k != '_extra']
10502 for table in tables:
10503 repr_fields = [f.name for f in fields if f.tablename == table] \
10504 if fields else [k for k in row[table].keys()
10505 if (hasattr(self.db[table], k) and
10506 isinstance(self.db[table][k], Field)
10507 and self.db[table][k].represent)]
10508 for field in repr_fields:
10509 row[table][field] = sqlhtml.represent(
10510 self.db[table][field], row[table][field], row[table])
10511 if self.compact and len(keys) == 1 and keys[0] != '_extra':
10512 return row[keys[0]]
10513 return row
10514
10515 - def as_list(self,
10516 compact=True,
10517 storage_to_dict=True,
10518 datetime_to_str=False,
10519 custom_types=None):
10520 """
10521 returns the data as a list or dictionary.
10522 :param storage_to_dict: when True returns a dict, otherwise a list(default True)
10523 :param datetime_to_str: convert datetime fields as strings (default False)
10524 """
10525 (oc, self.compact) = (self.compact, compact)
10526 if storage_to_dict:
10527 items = [item.as_dict(datetime_to_str, custom_types) for item in self]
10528 else:
10529 items = [item for item in self]
10530 self.compact = compact
10531 return items
10532
10533
10534 - def as_dict(self,
10535 key='id',
10536 compact=True,
10537 storage_to_dict=True,
10538 datetime_to_str=False,
10539 custom_types=None):
10540 """
10541 returns the data as a dictionary of dictionaries (storage_to_dict=True) or records (False)
10542
10543 :param key: the name of the field to be used as dict key, normally the id
10544 :param compact: ? (default True)
10545 :param storage_to_dict: when True returns a dict, otherwise a list(default True)
10546 :param datetime_to_str: convert datetime fields as strings (default False)
10547 """
10548
10549
10550 multi = False
10551 f = self.first()
10552 if f and isinstance(key, basestring):
10553 multi = any([isinstance(v, f.__class__) for v in f.values()])
10554 if (not "." in key) and multi:
10555
10556 def new_key():
10557 i = 0
10558 while True:
10559 yield i
10560 i += 1
10561 key_generator = new_key()
10562 key = lambda r: key_generator.next()
10563
10564 rows = self.as_list(compact, storage_to_dict, datetime_to_str, custom_types)
10565 if isinstance(key,str) and key.count('.')==1:
10566 (table, field) = key.split('.')
10567 return dict([(r[table][field],r) for r in rows])
10568 elif isinstance(key,str):
10569 return dict([(r[key],r) for r in rows])
10570 else:
10571 return dict([(key(r),r) for r in rows])
10572
10574 """
10575 export data to csv, the first line contains the column names
10576
10577 :param ofile: where the csv must be exported to
10578 :param null: how null values must be represented (default '<NULL>')
10579 :param delimiter: delimiter to separate values (default ',')
10580 :param quotechar: character to use to quote string values (default '"')
10581 :param quoting: quote system, use csv.QUOTE_*** (default csv.QUOTE_MINIMAL)
10582 :param represent: use the fields .represent value (default False)
10583 :param colnames: list of column names to use (default self.colnames)
10584 This will only work when exporting rows objects!!!!
10585 DO NOT use this with db.export_to_csv()
10586 """
10587 delimiter = kwargs.get('delimiter', ',')
10588 quotechar = kwargs.get('quotechar', '"')
10589 quoting = kwargs.get('quoting', csv.QUOTE_MINIMAL)
10590 represent = kwargs.get('represent', False)
10591 writer = csv.writer(ofile, delimiter=delimiter,
10592 quotechar=quotechar, quoting=quoting)
10593 colnames = kwargs.get('colnames', self.colnames)
10594 write_colnames = kwargs.get('write_colnames',True)
10595
10596 if write_colnames:
10597 writer.writerow(colnames)
10598
10599 def none_exception(value):
10600 """
10601 returns a cleaned up value that can be used for csv export:
10602 - unicode text is encoded as such
10603 - None values are replaced with the given representation (default <NULL>)
10604 """
10605 if value is None:
10606 return null
10607 elif isinstance(value, unicode):
10608 return value.encode('utf8')
10609 elif isinstance(value,Reference):
10610 return long(value)
10611 elif hasattr(value, 'isoformat'):
10612 return value.isoformat()[:19].replace('T', ' ')
10613 elif isinstance(value, (list,tuple)):
10614 return bar_encode(value)
10615 return value
10616
10617 for record in self:
10618 row = []
10619 for col in colnames:
10620 if not REGEX_TABLE_DOT_FIELD.match(col):
10621 row.append(record._extra[col])
10622 else:
10623 (t, f) = col.split('.')
10624 field = self.db[t][f]
10625 if isinstance(record.get(t, None), (Row,dict)):
10626 value = record[t][f]
10627 else:
10628 value = record[f]
10629 if field.type=='blob' and not value is None:
10630 value = base64.b64encode(value)
10631 elif represent and field.represent:
10632 value = field.represent(value)
10633 row.append(none_exception(value))
10634 writer.writerow(row)
10635
10636 - def xml(self,strict=False,row_name='row',rows_name='rows'):
10637 """
10638 serializes the table using sqlhtml.SQLTABLE (if present)
10639 """
10640
10641 if strict:
10642 ncols = len(self.colnames)
10643 return '<%s>\n%s\n</%s>' % (rows_name,
10644 '\n'.join(row.as_xml(row_name=row_name,
10645 colnames=self.colnames) for
10646 row in self), rows_name)
10647
10648 import sqlhtml
10649 return sqlhtml.SQLTABLE(self).xml()
10650
10651 - def as_xml(self,row_name='row',rows_name='rows'):
10652 return self.xml(strict=True, row_name=row_name, rows_name=rows_name)
10653
10654 - def as_json(self, mode='object', default=None):
10655 """
10656 serializes the rows to a JSON list or object with objects
10657 mode='object' is not implemented (should return a nested
10658 object structure)
10659 """
10660
10661 items = [record.as_json(mode=mode, default=default,
10662 serialize=False,
10663 colnames=self.colnames) for
10664 record in self]
10665
10666 if have_serializers:
10667 return serializers.json(items,
10668 default=default or
10669 serializers.custom_json)
10670 elif simplejson:
10671 return simplejson.dumps(items)
10672 else:
10673 raise RuntimeError("missing simplejson")
10674
10675
10676 as_csv = __str__
10677 json = as_json
10678
10679
10680
10681
10682
10683
10684 -def test_all():
10685 """
10686
10687 >>> if len(sys.argv)<2: db = DAL("sqlite://test.db")
10688 >>> if len(sys.argv)>1: db = DAL(sys.argv[1])
10689 >>> tmp = db.define_table('users',\
10690 Field('stringf', 'string', length=32, required=True),\
10691 Field('booleanf', 'boolean', default=False),\
10692 Field('passwordf', 'password', notnull=True),\
10693 Field('uploadf', 'upload'),\
10694 Field('blobf', 'blob'),\
10695 Field('integerf', 'integer', unique=True),\
10696 Field('doublef', 'double', unique=True,notnull=True),\
10697 Field('jsonf', 'json'),\
10698 Field('datef', 'date', default=datetime.date.today()),\
10699 Field('timef', 'time'),\
10700 Field('datetimef', 'datetime'),\
10701 migrate='test_user.table')
10702
10703 Insert a field
10704
10705 >>> db.users.insert(stringf='a', booleanf=True, passwordf='p', blobf='0A',\
10706 uploadf=None, integerf=5, doublef=3.14,\
10707 jsonf={"j": True},\
10708 datef=datetime.date(2001, 1, 1),\
10709 timef=datetime.time(12, 30, 15),\
10710 datetimef=datetime.datetime(2002, 2, 2, 12, 30, 15))
10711 1
10712
10713 Drop the table
10714
10715 >>> db.users.drop()
10716
10717 Examples of insert, select, update, delete
10718
10719 >>> tmp = db.define_table('person',\
10720 Field('name'),\
10721 Field('birth','date'),\
10722 migrate='test_person.table')
10723 >>> person_id = db.person.insert(name='Marco',birth='2005-06-22')
10724 >>> person_id = db.person.insert(name='Massimo',birth='1971-12-21')
10725
10726 commented len(db().select(db.person.ALL))
10727 commented 2
10728
10729 >>> me = db(db.person.id==person_id).select()[0] # test select
10730 >>> me.name
10731 'Massimo'
10732 >>> db.person[2].name
10733 'Massimo'
10734 >>> db.person(2).name
10735 'Massimo'
10736 >>> db.person(name='Massimo').name
10737 'Massimo'
10738 >>> db.person(db.person.name=='Massimo').name
10739 'Massimo'
10740 >>> row = db.person[2]
10741 >>> row.name == row['name'] == row['person.name'] == row('person.name')
10742 True
10743 >>> db(db.person.name=='Massimo').update(name='massimo') # test update
10744 1
10745 >>> db(db.person.name=='Marco').select().first().delete_record() # test delete
10746 1
10747
10748 Update a single record
10749
10750 >>> me.update_record(name="Max")
10751 <Row {'name': 'Max', 'birth': datetime.date(1971, 12, 21), 'id': 2}>
10752 >>> me.name
10753 'Max'
10754
10755 Examples of complex search conditions
10756
10757 >>> len(db((db.person.name=='Max')&(db.person.birth<'2003-01-01')).select())
10758 1
10759 >>> len(db((db.person.name=='Max')&(db.person.birth<datetime.date(2003,01,01))).select())
10760 1
10761 >>> len(db((db.person.name=='Max')|(db.person.birth<'2003-01-01')).select())
10762 1
10763 >>> me = db(db.person.id==person_id).select(db.person.name)[0]
10764 >>> me.name
10765 'Max'
10766
10767 Examples of search conditions using extract from date/datetime/time
10768
10769 >>> len(db(db.person.birth.month()==12).select())
10770 1
10771 >>> len(db(db.person.birth.year()>1900).select())
10772 1
10773
10774 Example of usage of NULL
10775
10776 >>> len(db(db.person.birth==None).select()) ### test NULL
10777 0
10778 >>> len(db(db.person.birth!=None).select()) ### test NULL
10779 1
10780
10781 Examples of search conditions using lower, upper, and like
10782
10783 >>> len(db(db.person.name.upper()=='MAX').select())
10784 1
10785 >>> len(db(db.person.name.like('%ax')).select())
10786 1
10787 >>> len(db(db.person.name.upper().like('%AX')).select())
10788 1
10789 >>> len(db(~db.person.name.upper().like('%AX')).select())
10790 0
10791
10792 orderby, groupby and limitby
10793
10794 >>> people = db().select(db.person.name, orderby=db.person.name)
10795 >>> order = db.person.name|~db.person.birth
10796 >>> people = db().select(db.person.name, orderby=order)
10797
10798 >>> people = db().select(db.person.name, orderby=db.person.name, groupby=db.person.name)
10799
10800 >>> people = db().select(db.person.name, orderby=order, limitby=(0,100))
10801
10802 Example of one 2 many relation
10803
10804 >>> tmp = db.define_table('dog',\
10805 Field('name'),\
10806 Field('birth','date'),\
10807 Field('owner',db.person),\
10808 migrate='test_dog.table')
10809 >>> db.dog.insert(name='Snoopy', birth=None, owner=person_id)
10810 1
10811
10812 A simple JOIN
10813
10814 >>> len(db(db.dog.owner==db.person.id).select())
10815 1
10816
10817 >>> len(db().select(db.person.ALL, db.dog.name,left=db.dog.on(db.dog.owner==db.person.id)))
10818 1
10819
10820 Drop tables
10821
10822 >>> db.dog.drop()
10823 >>> db.person.drop()
10824
10825 Example of many 2 many relation and Set
10826
10827 >>> tmp = db.define_table('author', Field('name'),\
10828 migrate='test_author.table')
10829 >>> tmp = db.define_table('paper', Field('title'),\
10830 migrate='test_paper.table')
10831 >>> tmp = db.define_table('authorship',\
10832 Field('author_id', db.author),\
10833 Field('paper_id', db.paper),\
10834 migrate='test_authorship.table')
10835 >>> aid = db.author.insert(name='Massimo')
10836 >>> pid = db.paper.insert(title='QCD')
10837 >>> tmp = db.authorship.insert(author_id=aid, paper_id=pid)
10838
10839 Define a Set
10840
10841 >>> authored_papers = db((db.author.id==db.authorship.author_id)&(db.paper.id==db.authorship.paper_id))
10842 >>> rows = authored_papers.select(db.author.name, db.paper.title)
10843 >>> for row in rows: print row.author.name, row.paper.title
10844 Massimo QCD
10845
10846 Example of search condition using belongs
10847
10848 >>> set = (1, 2, 3)
10849 >>> rows = db(db.paper.id.belongs(set)).select(db.paper.ALL)
10850 >>> print rows[0].title
10851 QCD
10852
10853 Example of search condition using nested select
10854
10855 >>> nested_select = db()._select(db.authorship.paper_id)
10856 >>> rows = db(db.paper.id.belongs(nested_select)).select(db.paper.ALL)
10857 >>> print rows[0].title
10858 QCD
10859
10860 Example of expressions
10861
10862 >>> mynumber = db.define_table('mynumber', Field('x', 'integer'))
10863 >>> db(mynumber).delete()
10864 0
10865 >>> for i in range(10): tmp = mynumber.insert(x=i)
10866 >>> db(mynumber).select(mynumber.x.sum())[0](mynumber.x.sum())
10867 45
10868
10869 >>> db(mynumber.x+2==5).select(mynumber.x + 2)[0](mynumber.x + 2)
10870 5
10871
10872 Output in csv
10873
10874 >>> print str(authored_papers.select(db.author.name, db.paper.title)).strip()
10875 author.name,paper.title\r
10876 Massimo,QCD
10877
10878 Delete all leftover tables
10879
10880 >>> DAL.distributed_transaction_commit(db)
10881
10882 >>> db.mynumber.drop()
10883 >>> db.authorship.drop()
10884 >>> db.author.drop()
10885 >>> db.paper.drop()
10886 """
10887
10888
10889
10890
10891 SQLField = Field
10892 SQLTable = Table
10893 SQLXorable = Expression
10894 SQLQuery = Query
10895 SQLSet = Set
10896 SQLRows = Rows
10897 SQLStorage = Row
10898 SQLDB = DAL
10899 GQLDB = DAL
10900 DAL.Field = Field
10901 DAL.Table = Table
10902
10903
10904
10905
10906
10907 -def geoPoint(x,y):
10908 return "POINT (%f %f)" % (x,y)
10909
10911 return "LINESTRING (%s)" % ','.join("%f %f" % item for item in line)
10912
10914 return "POLYGON ((%s))" % ','.join("%f %f" % item for item in line)
10915
10916
10917
10918
10919
10920 if __name__ == '__main__':
10921 import doctest
10922 doctest.testmod()
10923